r4501 - branches/scale_testing/mech_scripts
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-02 21:26:13 +0000 (Wed, 02 Feb 2011)
New Revision: 4501
Added:
branches/scale_testing/mech_scripts/process_times.py
Modified:
branches/scale_testing/mech_scripts/test_driver.py
Log:
Add busy_thread count to test_driver, include script for parsing custom
headers offline and generating CSV file.
Added: branches/scale_testing/mech_scripts/process_times.py
===================================================================
--- branches/scale_testing/mech_scripts/process_times.py (rev 0)
+++ branches/scale_testing/mech_scripts/process_times.py 2011-02-02 21:26:13 UTC (rev 4501)
@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+
+# make the main loop accept a directory, and loop over all the files
+# change the print out so it list the file name before dumping out the stats
+
+from datetime import *
+from dateutil import parser as dateparser
+import sys
+import os
+import csv
+import optparse
+
+def dump(stats):
+ try:
+ print stats[0], "\n", \
+ "accept:", stats[1], "\n", \
+ "to_service:", stats[2], "\n", \
+ "to_proc:", stats[3], "\n", \
+ "proc:", stats[4], "\n", \
+ "to_send:", stats[5], "\n", \
+ "total:", stats[6], "\n", \
+ "busy threads:", stats[7], "\n"
+ except:
+ print "truncated data\n"
+
+def to_secs(stats):
+ res = list()
+ for i in range(len(stats)):
+ # If it's not a time, just copy
+ if hasattr(stats[i], "seconds"):
+ res.append(stats[i].seconds + stats[i].microseconds / 1000000.0)
+ else:
+ res.append(stats[i])
+ return res
+
+def keep_sum(stats,values):
+ res = list()
+ # values has a label in the first field
+ for i in range(len(stats)):
+ res.append(stats[i] + values[i+1])
+ return res
+
+def get_zero_list():
+ # these match the stat[1-7] in the dump routine above.
+ # label is missing
+ return [0,0,0,0,0,0,0]
+
+def process_file(fname):
+
+ # symbolic names for fields in input line
+ link = 0
+ before_accept = 1
+ after_accept = 2
+ put_time = 3
+ after_get = 4
+ begin_service = 5
+ begin_proc = 6
+ end_proc = 7
+ send_headers = 8
+ busy_threads = 9
+
+ # start iteration count at 1, bump for blank lines in inputs.
+ # this is used for calculating averages from the sums later
+ iterations = 1
+
+ # Sum and average the whole file
+ sums = get_zero_list()
+ longest = [""]+sums # this will be the worst case get, with a label
+ count = 0
+
+ # Create a zero time delta
+ max = datetime.utcnow()
+ max = max - max
+
+ p = os.path.abspath(fname)
+ f = open(p, "r")
+ for line in f:
+
+ # Look for iteration marker
+ if len(line.strip()) == 0:
+ iterations += 1
+ continue
+
+ times = line.strip().split(",")
+
+ # handle files I collected before....
+ if len(times) < busy_threads+1:
+ for i in range(len(times), busy_threads+1):
+ times.append("0")
+
+ skip_line = False
+ for i in range(before_accept,send_headers+1):
+ if times[i] == "0":
+ # skip the record, bogus date info. Probably wrong web-server running
+ skip_line = True
+ break
+ times[i] = dateparser.parse(times[i])
+
+ # turn the thread count into a number
+ times[busy_threads] = int(times[busy_threads])
+
+ if skip_line:
+ continue
+
+ count += 1
+ accept = times[after_accept] - times[before_accept]
+ to_service = times[begin_service] - times[after_accept]
+ to_proc = times[begin_proc] - times[begin_service]
+ proc = times[end_proc] - times[begin_proc]
+ to_send = times[send_headers] - times[end_proc]
+ total = times[send_headers] - times[after_accept]
+ active_threads = times[busy_threads]
+
+ # First element is link name
+ stats = to_secs((times[0], accept, to_service, to_proc, proc, to_send, total, active_threads))
+ if total > max:
+ max = total
+ longest = stats
+
+ #dump(stats)
+ sums = keep_sum(sums, stats)
+
+ avg = list()
+ if count > 0:
+ for i in range(len(sums)):
+ avg.append(sums[i] / count)
+
+ print "**************** " + fname + " ****************"
+
+# print "longest:"
+# dump(longest)
+
+ dump(["average values"]+avg)
+
+ # Blank lines in the input file represent an iteration.
+ # Get average values for a page by counting and dividing by the number of iterations.
+ page_avg = list()
+ for i in range(len(sums)):
+ page_avg.append(sums[i] / iterations)
+
+ # The busy thread count should be an average thread count over the whole page file.
+ # Not a sum of average durations for code segments across mutiple gets, like the rest
+ # We already have this number.
+ try:
+ page_avg[len(avg)-1] = avg[len(avg)-1]
+ except:
+ pass
+
+ dump(["per page"]+page_avg)
+
+ return page_avg
+
+def main():
+
+ parser = optparse.OptionParser()
+ parser.add_option("--csvfile", dest="csvfile", default="")
+ parser.add_option("--threads", dest="threads", type=int, default=0)
+
+ opts, args = parser.parse_args()
+
+ if len(sys.argv) < 2:
+ print "Forgot the file, eh?"
+ return
+
+ # Allowing processing of a single page file, possibly with multiple iterations
+ # or an entire directory of page files, each with multiple iterations
+ file_list = []
+ if os.path.isdir(sys.argv[1]):
+ for i in os.listdir(sys.argv[1]):
+ file_list.append(os.path.join(sys.argv[1], i))
+ else:
+ file_list = [sys.argv[1]]
+
+ totals = get_zero_list()
+ for i in file_list:
+ page_values = process_file(i)
+ for i in range(len(page_values)):
+ totals[i] += page_values[i]
+
+ # Find average of all pages processed over multiple iterations.
+ for i in range(len(totals)):
+ totals[i] = totals[i] / len(file_list)
+
+ dump(["all pages"]+totals)
+
+ if opts.csvfile:
+ if not os.path.isfile(opts.csvfile):
+ db = csv.writer(open(opts.csvfile, "w+"))
+ db.writerow(("Threads", "accept", "to_service", "to_proc", "proc", "to_send", "total", "busy threads"))
+ else:
+ db = csv.writer(open(opts.csvfile, "a"))
+ db.writerow([opts.threads]+totals)
+
+if __name__ == "__main__":
+ main()
Modified: branches/scale_testing/mech_scripts/test_driver.py
===================================================================
--- branches/scale_testing/mech_scripts/test_driver.py 2011-02-02 15:40:51 UTC (rev 4500)
+++ branches/scale_testing/mech_scripts/test_driver.py 2011-02-02 21:26:13 UTC (rev 4501)
@@ -1,11 +1,8 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2010 Corey Goldberg (corey(a)goldb.org)
-# License: GNU LGPLv3
-#
-# This file is part of Multi-Mechanize
+#!/usr/bin/env python
+# make this take on option for header file output
+
import httplib
from time import time as time_time
from datetime import *
@@ -55,7 +52,7 @@
return name
def handle_custom_headers(self, curr_file, link, resp, insert_blank):
- fname = self.group + self.thread + "_" + os.path.basename(curr_file)
+ fname = os.path.join(self.outdir, self.group + self.thread + "_" + os.path.basename(curr_file))
if os.path.isfile(fname):
f = open(fname, "a")
if insert_blank:
@@ -66,7 +63,7 @@
f.write(link)
for i in ("page_before_accept", "page_after_accept", "page_put_time",
"page_after_get", "page_begin_service", "page_begin_proc",
- "page_end_proc","page_send_headers"):
+ "page_end_proc","page_send_headers","page_busy_threads"):
f.write(","+resp.getheader(i,"0"))
f.write("\n")
f.close()
@@ -79,6 +76,7 @@
# parse options on first invocation
if self.machine == None:
+
parser = optparse.OptionParser()
parser.add_option('--machine', dest='machine', type='str', default="localhost:45672")
parser.add_option('--file', dest='file', type='str', default="./links")
@@ -86,11 +84,13 @@
parser.add_option('--info', dest='info', action="store_true", default=False)
# combine these to create an output file name for web server internal stats
- parser.add_option('--group', dest='group', type='str', default="headers")
+ parser.add_option('--group', dest='group', type='str', default="headers")
parser.add_option('--thread', dest='thread', type='str', default="")
+ parser.add_option('--outdir', dest='outdir', type='str', default=".")
if type(self.script_options) == str:
self.script_options = self.script_options.split()
+
opts, args = parser.parse_args(self.script_options)
self.machine = opts.machine
@@ -99,6 +99,9 @@
self.print_info = opts.info
self.group = opts.group
self.thread = opts.thread
+ self.outdir = opts.outdir
+ if self.outdir != "." and not os.path.isdir(self.outdir):
+ os.mkdir(self.outdir)
curr_file = self.next_file()
f = open(curr_file, "r")
13 years, 3 months
r4500 - in trunk/cumin/python/cumin: . grid inventory messaging usergrid
by eallen@fedoraproject.org
Author: eallen
Date: 2011-02-02 15:40:51 +0000 (Wed, 02 Feb 2011)
New Revision: 4500
Modified:
trunk/cumin/python/cumin/grid/job.py
trunk/cumin/python/cumin/grid/negotiator.py
trunk/cumin/python/cumin/grid/negotiator.strings
trunk/cumin/python/cumin/grid/pool.py
trunk/cumin/python/cumin/grid/scheduler.py
trunk/cumin/python/cumin/grid/slot.py
trunk/cumin/python/cumin/grid/submission.py
trunk/cumin/python/cumin/grid/submitter.py
trunk/cumin/python/cumin/inventory/system.py
trunk/cumin/python/cumin/main.py
trunk/cumin/python/cumin/messaging/binding.py
trunk/cumin/python/cumin/messaging/brokerlink.py
trunk/cumin/python/cumin/messaging/connection.py
trunk/cumin/python/cumin/messaging/exchange.py
trunk/cumin/python/cumin/messaging/queue.py
trunk/cumin/python/cumin/messaging/subscription.py
trunk/cumin/python/cumin/model.py
trunk/cumin/python/cumin/objectselector.py
trunk/cumin/python/cumin/objectselector.strings
trunk/cumin/python/cumin/stat.strings
trunk/cumin/python/cumin/usergrid/widgets.py
trunk/cumin/python/cumin/widgets.py
Log:
BZ 673187 - 2.0 Feature - Export table data as csv
Modified: trunk/cumin/python/cumin/grid/job.py
===================================================================
--- trunk/cumin/python/cumin/grid/job.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/grid/job.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -101,20 +101,34 @@
return self.frame.get_title(session)
class JobSummariesAdapter(ObjectQmfAdapter):
+ def get_qmf_response(self, values):
+ try:
+ submission = values["obj"]
+ except:
+ return None
+
+ return self.app.model.get_submission_job_summaries(submission)
+
def get_data(self, values, options):
- first_index = options.offset
- last_index = options.offset + options.limit
+ results = self.get_qmf_response(values)
- submission = values["obj"]
- results = self.app.model.get_submission_job_summaries(submission)
+ if not results:
+ return []
rows = self.process_list_results(results.data)
- # sort the entore results set
+ # sort the entire results set
rows = self.sort_rows(rows, options)
+
+ # return only the current page
+ first_index = options.offset
+ last_index = len(rows)
+ if options.limit:
+ last_index = options.offset + options.limit
+
page = rows[first_index:last_index]
- return page, results
+ return page
def process_record(self, key, record):
field_data = list()
@@ -197,6 +211,7 @@
def __init__(self, app, name, submission):
cls = app.model.com_redhat_cumin_grid.JobSummary
+ self.defer_enabled = True
super(JobSelector, self).__init__(app, name, cls)
self.table.adapter = JobSummariesAdapter(app, cls)
@@ -222,6 +237,8 @@
JobSelectionRelease(app, self, "released")
JobSelectionRemove(app, self, "removed")
+ self.enable_csv_export(submission)
+
def create_table(self, app, name, cls):
return JobSelectorTable(app, name, cls)
@@ -257,8 +274,6 @@
super(JobSelectorTable, self).__init__(app, name, cls)
self.error_tmpl = WidgetTemplate(self, "error_html")
- self.qmf_response = Attribute(app, "qmf_response")
- self.add_attribute(self.qmf_response)
def init_ids(self, app, cls):
item = Parameter(app, "item")
@@ -284,24 +299,29 @@
values = self.get_data_values(session)
options = self.get_data_options(session)
- results, qmf_response = self.adapter.get_data(values, options)
- self.qmf_response.set(session, qmf_response)
- return results
+ return self.adapter.get_data(values, options)
def is_exception(self, session):
- self.get_data(session)
- qmf_response = self.qmf_response.get(session)
+ values = self.get_data_values(session)
+ qmf_response = self.adapter.get_qmf_response(values)
+
+ if not qmf_response:
+ return True
+
return qmf_response.exception and True or False
def do_render(self, session):
# determine if there is an error getting the data
# and show an error message
- self.get_data(session)
- qmf_response = self.qmf_response.get(session)
- if qmf_response.exception:
- writer = Writer()
+ values = self.get_data_values(session)
+ qmf_response = self.adapter.get_qmf_response(values)
+ if not qmf_response:
+ m = ""
+ elif qmf_response.exception:
e = qmf_response.exception
m = len(e.args) > 0 and e.args[0] or ""
+ if (not qmf_response) or qmf_response.exception:
+ writer = Writer()
msg = "The data is not available at this time. <br/>%s" % m
self.error_tmpl.render(writer, session, msg)
return writer.to_string()
Modified: trunk/cumin/python/cumin/grid/negotiator.py
===================================================================
--- trunk/cumin/python/cumin/grid/negotiator.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/grid/negotiator.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -63,6 +63,8 @@
#self.start = DaemonSelectionStart(app, self, "NEGOTIATOR")
#self.stop = DaemonSelectionStop(app, self, "NEGOTIATOR")
+ self.enable_csv_export(pool)
+
class QmfGroupColumn(ItemTableColumn):
def __init__(self, app, name, getter, negotiator, task):
super(QmfGroupColumn, self).__init__(app, name)
@@ -499,7 +501,7 @@
self.original_values = ListParameter(app, "original_values", original_value)
self.add_parameter(self.original_values)
- self.defer_enabled = True
+ #self.defer_enabled = True
def render_group_name(self, session, group):
return group
Modified: trunk/cumin/python/cumin/grid/negotiator.strings
===================================================================
--- trunk/cumin/python/cumin/grid/negotiator.strings 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/grid/negotiator.strings 2011-02-02 15:40:51 UTC (rev 4500)
@@ -106,6 +106,37 @@
</form>
<script type="text/javascript">
<![CDATA[
+
+cumin.initializeGroupForm = function () {
+ var elem = $("{id}").elements[0];
+ elem.focus();
+ cumin.init_sliders("{chart_id}");
+
+ var myFormValidator = new FormValidator($('{id}'), {
+ onElementPass: function (el) {
+ var val = el.value.toFloat();
+ var sliderDiv = el.getParent("tr").getElement(".slider");
+ if (sliderDiv) {
+ var myslider = sliderDiv.retrieve('myslider');
+ if (myslider)
+ myslider.set(val * 100);
+ }
+ },
+ onElementFail: function (el, validators) {
+ //alert('failed');
+ },
+ evaluateOnSubmit: false,
+ evaluateFieldsOnBlur: true
+ });
+ myFormValidator.add('IsPercent', {
+ errorMsg: 'Values must be between 0 and 100.',
+ test: function(element) {
+ var val = element.get('value').toFloat();
+ return ((val >= 0) && (val <= 100))
+ }
+ });
+}
+
(function() {
wooly.addPageUpdateListener( cumin.initializeGroupForm );
}())
@@ -252,40 +283,7 @@
<div class="deferredSpacer">Loading...</div>
</div>
</form>
-<script type="text/javascript">
-<![CDATA[
-cumin.initializeGroupForm = function () {
- var elem = $("{id}").elements[0];
- elem.focus();
- cumin.init_sliders("{chart_id}");
- var myFormValidator = new FormValidator($('{id}'), {
- onElementPass: function (el) {
- var val = el.value.toFloat();
- var sliderDiv = el.getParent("tr").getElement(".slider");
- if (sliderDiv) {
- var myslider = sliderDiv.retrieve('myslider');
- if (myslider)
- myslider.set(val * 100);
- }
- },
- onElementFail: function (el, validators) {
- //alert('failed');
- },
- evaluateOnSubmit: false,
- evaluateFieldsOnBlur: true
- });
- myFormValidator.add('IsPercent', {
- errorMsg: 'Values must be between 0 and 100.',
- test: function(element) {
- var val = element.get('value').toFloat();
- return ((val >= 0) && (val <= 100))
- }
- });
-}
-]]>
-</script>
-
[EditStaticQuotaForm.css]
form.StaticQuotaForm {
width: 30em;
Modified: trunk/cumin/python/cumin/grid/pool.py
===================================================================
--- trunk/cumin/python/cumin/grid/pool.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/grid/pool.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -85,7 +85,7 @@
overview = PoolOverview(app, "overview", self.object)
self.view.add_tab(overview)
- submissions = PoolSubmissionSchedulerSelector(app, "submissions", self.object)
+ submissions = PoolSubmissionJoinSelector(app, "pool_submissions", self.object)
self.view.add_tab(submissions)
slots = SlotSelector(app, "slots", self.object)
Modified: trunk/cumin/python/cumin/grid/scheduler.py
===================================================================
--- trunk/cumin/python/cumin/grid/scheduler.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/grid/scheduler.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -47,7 +47,7 @@
super(SchedulerPoolSubmissionSelector, self).__init__(app, name, pool)
self.add_reference_filter(scheduler, self.cls.jobserverRef)
- #self.enable_csv_export(pool, scheduler)
+ self.enable_csv_export(pool, scheduler)
def create_table(self, app, name, cls):
return self.JoinTable(app, name, cls)
@@ -105,7 +105,7 @@
#self.start = DaemonSelectionStart(app, self, "SCHEDD")
#self.stop = DaemonSelectionStop(app, self, "SCHEDD")
- #self.enable_csv_export(pool)
+ self.enable_csv_export(pool)
class SchedulerGeneralStatSet(StatSet):
def __init__(self, app, name, object):
Modified: trunk/cumin/python/cumin/grid/slot.py
===================================================================
--- trunk/cumin/python/cumin/grid/slot.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/grid/slot.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -34,6 +34,8 @@
self.add_attribute_column(cls.State)
self.add_attribute_column(cls.LoadAvg)
+ self.enable_csv_export(pool)
+
class SlotFrame(ObjectFrame):
def __init__(self, app, name):
cls = app.model.com_redhat_grid.Slot
Modified: trunk/cumin/python/cumin/grid/submission.py
===================================================================
--- trunk/cumin/python/cumin/grid/submission.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/grid/submission.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -105,9 +105,9 @@
pool = self.parent.pool.get(session)
self.form.pool_name.set(form_session, pool.Pool)
-class PoolSubmissionSchedulerSelector(PoolSubmissionSelector):
+class PoolSubmissionJoinSelector(PoolSubmissionSelector):
def __init__(self, app, name, pool):
- super(PoolSubmissionSchedulerSelector, self).__init__(app, name, pool)
+ super(PoolSubmissionJoinSelector, self).__init__(app, name, pool)
scheduler = app.model.com_redhat_grid.Scheduler
self.SchedulerJoin(app, self.table.adapter.query, self.cls.sql_table,
@@ -117,6 +117,8 @@
col = self.SchedulerColumn(app, "Scheduler", scheduler.Name, scheduler._id, frame)
self.insert_column(3, col)
+ self.enable_csv_export(pool)
+
class SchedulerColumn(ObjectLinkColumn):
def render_header_content(self, session):
return self.name
@@ -125,7 +127,7 @@
""" Connects Submission->jobserverRef->schedulerRef->Scheduler """
def __init__(self, app, query, table, this, that):
- super(PoolSubmissionSchedulerSelector.SchedulerJoin, self).__init__(query, table, this, that)
+ super(PoolSubmissionJoinSelector.SchedulerJoin, self).__init__(query, table, this, that)
self.adapter = self.SchedulerData(app)
@@ -141,7 +143,7 @@
jobserver = app.model.com_redhat_grid.JobServer
scheduler = app.model.com_redhat_grid.Scheduler
- super(PoolSubmissionSchedulerSelector.SchedulerJoin.SchedulerData, self).__init__(app, jobserver)
+ super(PoolSubmissionJoinSelector.SchedulerJoin.SchedulerData, self).__init__(app, jobserver)
self.add_join(scheduler, jobserver.schedulerRef, scheduler._id)
Modified: trunk/cumin/python/cumin/grid/submitter.py
===================================================================
--- trunk/cumin/python/cumin/grid/submitter.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/grid/submitter.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -52,6 +52,8 @@
self.add_search_filter(cls.Name)
+ self.enable_csv_export(scheduler)
+
class SubmitterGeneralStatSet(StatSet):
def __init__(self, app, name, object):
super(SubmitterGeneralStatSet, self).__init__(app, name, object)
Modified: trunk/cumin/python/cumin/inventory/system.py
===================================================================
--- trunk/cumin/python/cumin/inventory/system.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/inventory/system.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -32,6 +32,8 @@
self.add_search_filter(cls.nodeName)
+ self.enable_csv_export()
+
class SystemFrame(ObjectFrame):
def __init__(self, app, name):
cls = app.model.com_redhat_sesame.Sysimage
Modified: trunk/cumin/python/cumin/main.py
===================================================================
--- trunk/cumin/python/cumin/main.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/main.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -106,12 +106,14 @@
self.form_page = CuminFormPage(self, "form.html")
self.add_page(self.form_page)
- self.add_page(CallPage(self, "call.xml"))
self.add_page(StatChartPage(self, "stats.png"))
self.add_page(StatStackedPage(self, "stacked.png"))
self.add_page(StatFlashPage(self, "chart.json"))
self.add_page(FlashFullPage(self, "flashpage.html"))
+ self.export_page = CuminExportPage(self, "csv")
+ self.add_page(self.export_page)
+
self.resource_page.protected = False
def start(self):
Modified: trunk/cumin/python/cumin/messaging/binding.py
===================================================================
--- trunk/cumin/python/cumin/messaging/binding.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/messaging/binding.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -158,6 +158,8 @@
self.add_search_filter(binding.bindingKey)
+ self.enable_csv_export()
+
class Exchange(ObjectLinkColumn):
def render_header_content(self, session):
return "Exchange"
Modified: trunk/cumin/python/cumin/messaging/brokerlink.py
===================================================================
--- trunk/cumin/python/cumin/messaging/brokerlink.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/messaging/brokerlink.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -77,7 +77,7 @@
self.remove = BrokerLinkSelectionRemove(app, self)
- # Address column XXX
+ self.enable_csv_export(vhost)
class BrokerLinkSelectionRemove(ObjectSelectorTask):
def get_title(self, session):
Modified: trunk/cumin/python/cumin/messaging/connection.py
===================================================================
--- trunk/cumin/python/cumin/messaging/connection.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/messaging/connection.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -78,6 +78,8 @@
self.close = ConnectionSelectionClose(app, self)
+ self.enable_csv_export(vhost)
+
class ConnectionSelectionClose(ObjectSelectorTask):
def get_title(self, session):
return "Close"
Modified: trunk/cumin/python/cumin/messaging/exchange.py
===================================================================
--- trunk/cumin/python/cumin/messaging/exchange.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/messaging/exchange.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -29,7 +29,7 @@
self.overview = ExchangeOverview(app, "overview", self.object)
self.view.add_tab(self.overview)
- self.bindings = ExchangeBindingSelector(app, "bindings", self.object)
+ self.bindings = ExchangeBindingSelector(app, "ex_bindings", self.object)
self.view.add_tab(self.bindings)
self.remove = ExchangeRemove(app, self)
@@ -83,6 +83,8 @@
self.remove = ExchangeSelectionRemove(app, self)
+ self.enable_csv_export(vhost)
+
class ExchangeNameColumn(ObjectLinkColumn):
def render_cell_content(self, session, record):
return record[self.field.index] or "Default exchange"
Modified: trunk/cumin/python/cumin/messaging/queue.py
===================================================================
--- trunk/cumin/python/cumin/messaging/queue.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/messaging/queue.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -32,7 +32,7 @@
self.overview = QueueOverview(app, "overview", self.object)
self.view.add_tab(self.overview)
- self.bindings = QueueBindingSelector(app, "bindings", self.object)
+ self.bindings = QueueBindingSelector(app, "q_bindings", self.object)
self.view.add_tab(self.bindings)
self.subscriptions = SubscriptionSelector \
@@ -84,6 +84,8 @@
self.remove = QueueSelectionRemove(app, self)
self.purge = QueueSelectionPurge(app, self)
+ self.enable_csv_export(vhost)
+
class QueueSelectionRemove(ObjectSelectorTask):
def get_title(self, session):
return "Remove"
Modified: trunk/cumin/python/cumin/messaging/subscription.py
===================================================================
--- trunk/cumin/python/cumin/messaging/subscription.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/messaging/subscription.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -23,6 +23,8 @@
self.add_search_filter(cls.name)
+ self.enable_csv_export(queue)
+
class SubscriptionFrame(ObjectFrame):
def __init__(self, app, name):
cls = app.model.org_apache_qpid_broker.Subscription
Modified: trunk/cumin/python/cumin/model.py
===================================================================
--- trunk/cumin/python/cumin/model.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/model.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -1133,7 +1133,8 @@
def completion(status, data):
self.status = status
try:
- self.data = data["Value"]
+ #self.data = data["Value"]
+ self.data = "MSG, GRID, MGMT, RT, GRID.SUB_1, GRID.SUB_1.SUB_A"
except KeyError:
pass
@@ -1173,6 +1174,12 @@
for config in self.configs:
action = FetchRawConfigSet(self.model.app)
raw_configs = action.execute(self.negotiator, self.configs[config], config+"_")
+ for group in raw_configs:
+ qmfc = raw_configs[group]
+ qmfc.data = {'Value': 0.1}
+ qmfc.error = None
+ qmfc.got_data = True
+ qmfc.status = "OK"
self.data[config] = raw_configs
def delete(self):
Modified: trunk/cumin/python/cumin/objectselector.py
===================================================================
--- trunk/cumin/python/cumin/objectselector.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/objectselector.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -1,3 +1,8 @@
+from objectframe import *
+from objecttask import *
+from qmfadapter import *
+from sqladapter import *
+from util import *
from wooly import *
from wooly.datatable import *
from wooly.forms import *
@@ -3,9 +8,4 @@
from wooly.widgets import *
-from objectframe import *
-from objecttask import *
-from sqladapter import *
-from qmfadapter import *
-from util import *
strings = StringCatalog(__file__)
@@ -102,6 +102,52 @@
def render_title(self, session):
return "%ss" % self.cls._title
+class CsvExporter(Widget):
+ def __init__(self, app, name, args, selector):
+ super(CsvExporter, self).__init__(app, name)
+
+ # 0 or more RosemaryAttributes that need to be set
+ # for the selector's table to get it's data
+ self.objects = list(args)
+
+ self.selector = selector
+
+ # avoid name collisions on csv page
+ if self.name in self.app.export_page.modes.children_by_name:
+ self.name = self.name + ".1"
+
+ self.app.export_page.modes.add_mode(self)
+
+ def render(self, session):
+ table = self.selector.table
+
+ values = table.get_data_values(session)
+ options = table.get_data_options(session)
+ # get all the records, starting at the beginning
+ options.limit = None
+ options.offset = 0
+
+ data = table.adapter.get_data(values, options)
+
+ writer = Writer()
+
+ for record in data:
+ writer.write(",".join(self.render_csv_cells(session, record)))
+ writer.write("\n")
+
+ return writer.to_string()
+
+ def render_csv_cells(self, session, record):
+ cells = list()
+ for column in self.selector.table.get_visible_columns(session):
+ # don't output the checkbox column since it's just a database id
+ if column.name != "id":
+ value = column.cell.render_content(session, record)
+ value = value is not None and str(value).replace(",", "<comma>") or ""
+ cells.append(value)
+
+ return cells
+
class ObjectTableColumn(DataTableColumn):
def __init__(self, app, name, attr):
super(ObjectTableColumn, self).__init__(app, name)
@@ -126,7 +172,7 @@
self.field = ObjectSqlField(self.table.adapter, self.attr)
class ObjectSelector(Form):
- def __init__(self, app, name, cls):
+ def __init__(self, app, name, cls, object=None):
super(ObjectSelector, self).__init__(app, name)
self.update_enabled = False
@@ -149,6 +195,8 @@
self.tasks = list()
+ self.export = None
+
def init(self):
super(ObjectSelector, self).init()
@@ -163,6 +211,14 @@
def create_table(self, app, name, cls):
return ObjectSelectorTable(app, name, cls)
+ def enable_csv_export(self, *args):
+ assert self.table
+
+ exporter = CsvExporter(self.app, self.name + "_csv", args, self)
+
+ self.export = ExportButton(self.app, "export", args, exporter, self.cls._title)
+ self.add_child(self.export)
+
def add_search_filter(self, this):
search = StringInput(self.app, "search")
search.param.default = ""
@@ -188,6 +244,10 @@
def render_title(self, session):
return self.table.render_title(session)
+ def render_export(self, session):
+ if self.export:
+ return self.export.render(session)
+
def get_data_values(self, session):
return self.table.get_data_values(session)
@@ -410,3 +470,19 @@
if parts[0] == "Monitor" and parts[1] == "Self":
return " ".join(parts[2:])
return title
+
+class ExportButton(Widget):
+ def __init__(self, app, name, args, exporter, file_name):
+ super(ExportButton, self).__init__(app, name)
+
+ self.object_attributes = list(args)
+ self.file_name = file_name
+ self.exporter = exporter
+
+ def render_href(self, session):
+ page = self.app.export_page
+ export_session = wooly.Session(page)
+
+ page.modes.show_child(export_session, self.exporter)
+ page.set_parameters(export_session, session, self.object_attributes, self.file_name)
+ return escape_entity(export_session.marshal())
Modified: trunk/cumin/python/cumin/objectselector.strings
===================================================================
--- trunk/cumin/python/cumin/objectselector.strings 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/objectselector.strings 2011-02-02 15:40:51 UTC (rev 4500)
@@ -44,6 +44,7 @@
{switches}
<form id="{id}.form" method="post" action="?">
+ {export}
{filters}
{buttons}
@@ -83,3 +84,6 @@
<ul>{widgets}</ul>
<input type="submit" value="Search"/>
</div><div style="clear:both;"></div>
+
+[ExportButton.html]
+<a href="{href}"><img src="resource?name=csv.png" alt="save as csv" title="Save table as CSV file" /></a>
Modified: trunk/cumin/python/cumin/stat.strings
===================================================================
--- trunk/cumin/python/cumin/stat.strings 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/stat.strings 2011-02-02 15:40:51 UTC (rev 4500)
@@ -111,7 +111,7 @@
font-size: 0.9em;
margin: 0 0 1.5em 0;
position: relative;
- width: 30em;
+ width: 31em;
}
div.StatValueChart div.duration {
Modified: trunk/cumin/python/cumin/usergrid/widgets.py
===================================================================
--- trunk/cumin/python/cumin/usergrid/widgets.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/usergrid/widgets.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -124,6 +124,8 @@
link = TaskLink(app, "vm_job_submit", app.grid.vm_job_submit)
self.links.add_child(link)
+ self.enable_csv_export(user)
+
class UserJobStatSet(NewStatSet):
def __init__(self, app, name, user):
super(UserJobStatSet, self).__init__(app, name)
Modified: trunk/cumin/python/cumin/widgets.py
===================================================================
--- trunk/cumin/python/cumin/widgets.py 2011-02-02 15:36:40 UTC (rev 4499)
+++ trunk/cumin/python/cumin/widgets.py 2011-02-02 15:40:51 UTC (rev 4500)
@@ -1426,49 +1426,74 @@
url = form.return_url.get(session)
return url
-class CallPage(Page):
+class CuminExportPage(CsvPage):
def __init__(self, app, name):
- super(CallPage, self).__init__(app, name)
+ super(CuminExportPage, self).__init__(app, name)
- self.__class = CuminClassParameter(app, "class")
- self.add_parameter(self.__class)
+ self.modes = ModeSet(app, "modes")
+ self.add_child(self.modes)
- self.__id = IntegerParameter(app, "id")
- self.add_parameter(self.__id)
+ self.filtered = Parameter(app, "filtered")
+ self.add_parameter(self.filtered)
- self.__method = Parameter(app, "method")
- self.add_parameter(self.__method)
+ self.file_name = Parameter(app, "file")
+ self.add_parameter(self.file_name)
- self.__args = Parameter(app, "xargs")
- self.add_parameter(self.__args)
+ cls = Parameter(app, "class")
+ self.classes = ListParameter(app, "classes", cls)
+ self.add_parameter(self.classes)
- def get_content_type(self, session):
- return Page.xml_content_type
+ package = Parameter(app, "package")
+ self.packages = ListParameter(app, "packages", package)
+ self.add_parameter(self.packages)
- def do_render(self, session):
+ agent = Parameter(app, "agent")
+ self.agents = ListParameter(app, "agents", agent)
+ self.add_parameter(self.agents)
+
+ def render_content(self, session, *args):
writer = Writer()
- writer.write(Page.xml_1_0_declaration)
- cls = self.__class.get(session)
- id = self.__id.get(session)
- xargs = self.__args.get(session)
+ mode = self.modes.mode.get(session)
+ objects = self.get_objects(session)
+ for obj, mobj in zip(objects, mode.objects):
+ mobj.set(session, obj)
- if cls:
- object = cls.mint_class.get(id)
- method = self.__method.get(session)
- for action in cls.actions:
- if action.name == method:
- args = xargs and xargs.split("&") or list()
- data = action.get_xml_response(session, object, *args)
- self.write_xml(writer, data)
+ writer.write(mode.render(session))
return writer.to_string()
- def write_xml(self, writer, data):
- writer.write("<data>")
- writer.write(data)
- writer.write("</data>")
+ def do_process(self, session):
+ session.cursor = self.app.database.get_read_cursor()
+ def get_objects(self, session):
+ classes = self.classes.get(session)
+ packages = self.packages.get(session)
+ agents = self.agents.get(session)
+
+ objects = list()
+ cursor = self.app.database.get_read_cursor()
+
+ for cls, package, agent in zip(classes, packages, agents):
+ rosemary_package = self.app.model._packages_by_name[package]
+ rosemary_cls = rosemary_package._classes_by_name[cls]
+ objects.append(rosemary_cls.get_object(cursor, _qmf_agent_id=agent))
+
+ return objects
+
+ def get_file_name(self, session):
+ name = self.file_name.get(session)
+ return "%s.csv" % name
+
+ def set_parameters(self, nsession, session, attribs, name):
+ self.file_name.set(nsession, name)
+ for attrib in attribs:
+ obj = attrib.get(session)
+ cls = obj._class
+ self.packages.add(nsession, cls._package._name)
+ self.classes.add(nsession, cls._name)
+ self.agents.add(nsession, obj._qmf_agent_id)
+
class IncrementalSearchInput(StringInput, ItemSet):
def __init__(self, app, name, field_param):
super(IncrementalSearchInput, self).__init__(app, name)
13 years, 3 months
r4499 - trunk/cumin/resources
by eallen@fedoraproject.org
Author: eallen
Date: 2011-02-02 15:36:40 +0000 (Wed, 02 Feb 2011)
New Revision: 4499
Added:
trunk/cumin/resources/csv.png
Log:
new icon for exporting tables to csv
Added: trunk/cumin/resources/csv.png
===================================================================
(Binary files differ)
Property changes on: trunk/cumin/resources/csv.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
13 years, 3 months
r4498 - trunk/wooly/python/wooly
by eallen@fedoraproject.org
Author: eallen
Date: 2011-02-02 15:35:17 +0000 (Wed, 02 Feb 2011)
New Revision: 4498
Modified:
trunk/wooly/python/wooly/__init__.py
trunk/wooly/python/wooly/server.py
Log:
Allow pages to send extra headers in response
Modified: trunk/wooly/python/wooly/__init__.py
===================================================================
--- trunk/wooly/python/wooly/__init__.py 2011-02-02 15:33:14 UTC (rev 4497)
+++ trunk/wooly/python/wooly/__init__.py 2011-02-02 15:35:17 UTC (rev 4498)
@@ -447,6 +447,9 @@
def get_content_type(self, session):
raise Exception("Not implemented")
+ def get_extra_headers(self, session):
+ return []
+
def get_cache_control(self, session):
return None
Modified: trunk/wooly/python/wooly/server.py
===================================================================
--- trunk/wooly/python/wooly/server.py 2011-02-02 15:33:14 UTC (rev 4497)
+++ trunk/wooly/python/wooly/server.py 2011-02-02 15:35:17 UTC (rev 4498)
@@ -136,6 +136,12 @@
headers.append(("Content-Length", content_length))
headers.append(("Content-Type", content_type))
+ # additional headers take the form of an array of tuples
+ # ie [("Content-description", "File Transfer")]
+ content_extra_headers = page.get_extra_headers(session)
+ for extra_header in content_extra_headers:
+ headers.append(extra_header)
+
cache = page.get_cache_control(session)
if cache:
13 years, 3 months
r4497 - trunk/wooly/python/wooly
by eallen@fedoraproject.org
Author: eallen
Date: 2011-02-02 15:33:14 +0000 (Wed, 02 Feb 2011)
New Revision: 4497
Modified:
trunk/wooly/python/wooly/pages.py
Log:
Add new page type for exporting table values
Modified: trunk/wooly/python/wooly/pages.py
===================================================================
--- trunk/wooly/python/wooly/pages.py 2011-02-01 22:34:51 UTC (rev 4496)
+++ trunk/wooly/python/wooly/pages.py 2011-02-02 15:33:14 UTC (rev 4497)
@@ -363,3 +363,19 @@
if resource:
return resource.read()
+
+class CsvPage(WidgetPage):
+ def get_content_type(self, session):
+ return "text/csv"
+
+ def get_file_name(self, session):
+ return "cumin.csv"
+
+ def get_extra_headers(self, session):
+ file_name = self.get_file_name(session)
+ return [('Content-Disposition', "attachment; filename=\"%s\"" % file_name),
+ ("Content-description", "File Transfer")]
+
+ def do_render(self, session, *args):
+ csv = self.render_content(session, *args)
+ return csv and csv or " "
13 years, 3 months
r4496 - in branches/scale_testing/wooly/python/wooly: . wsgiserver
by tmckay@fedoraproject.org
Author: tmckay
Date: 2011-02-01 22:34:51 +0000 (Tue, 01 Feb 2011)
New Revision: 4496
Modified:
branches/scale_testing/wooly/python/wooly/server.py
branches/scale_testing/wooly/python/wooly/wsgiserver/__init__.py
Log:
Add custom headers to track time stamps throughout page request processing.
Client can pull out headers from response and get internal metrics.
Modified: branches/scale_testing/wooly/python/wooly/server.py
===================================================================
--- branches/scale_testing/wooly/python/wooly/server.py 2011-01-31 14:43:25 UTC (rev 4495)
+++ branches/scale_testing/wooly/python/wooly/server.py 2011-02-01 22:34:51 UTC (rev 4496)
@@ -1,5 +1,7 @@
import socket
+from datetime import *
+
from util import *
from wooly import *
from wsgiserver import CherryPyWSGIServer
@@ -105,14 +107,26 @@
f.write("False,")
f.write(link+"\n")
- def service_request(self, env, response):
+ def service_request(self, env, response, scale_times=None):
+ begin_service = datetime.utcnow()
log.info("Request %s %s", env["REQUEST_METHOD"], env["REQUEST_URI"])
- self.log_link(env["REQUEST_URI"], self.get_last_requested(env) != None)
+ #self.log_link(env["REQUEST_URI"], self.get_last_requested(env) != None)
page = self.get_page(env)
+ before = datetime.utcnow()
if page:
status, headers, content = self.service_page_request(page, env)
+ after = datetime.utcnow()
+ if scale_times:
+ headers.append(("page_before_accept", str(scale_times.tick_before_accept)))
+ headers.append(("page_after_accept", str(scale_times.tick_after_accept)))
+ headers.append(("page_put_time", str(scale_times.tick_at_put)))
+ headers.append(("page_after_get", str(scale_times.thread_after_get)))
+ headers.append(("page_begin_service", str(begin_service)))
+ headers.append(("page_begin_proc", str(before)))
+ headers.append(("page_end_proc", str(after)))
+ headers.append(("page_busy_threads", str(scale_times.busy_threads)))
else:
status = "404 Not Found"
headers = ()
Modified: branches/scale_testing/wooly/python/wooly/wsgiserver/__init__.py
===================================================================
--- branches/scale_testing/wooly/python/wooly/wsgiserver/__init__.py 2011-01-31 14:43:25 UTC (rev 4495)
+++ branches/scale_testing/wooly/python/wooly/wsgiserver/__init__.py 2011-02-01 22:34:51 UTC (rev 4496)
@@ -91,7 +91,8 @@
import errno
-import cProfile
+#import cProfile
+from datetime import datetime
def plat_specific_errors(*errnames):
"""Return error numbers for all errors in errnames on this platform.
@@ -668,7 +669,7 @@
env = self.get_version_specific_environ()
#for k, v in sorted(env.items()):
# print(k, '=', v)
- response = self.wsgi_app(env, self.start_response)
+ response = self.wsgi_app(env, self.start_response, self.scale_times)
try:
for chunk in response:
# "The start_response callable must not actually transmit
@@ -793,6 +794,9 @@
def send_headers(self):
"""Assert, process, and send the HTTP response message-headers."""
+
+ self.outheaders.append(("page_send_headers", str(datetime.utcnow())))
+
hkeys = [key.lower() for key, value in self.outheaders]
status = int(self.status[:3])
@@ -1239,7 +1243,8 @@
req = None
req = self.RequestHandlerClass(
self.rfile, self.wfile, self.environ, self.wsgi_app)
-
+ req.scale_times = self.scale_times
+
# This order of operations should guarantee correct pipelining.
req.parse_request()
if not req.ready:
@@ -1392,6 +1397,12 @@
#cProfile.runctx('profWrapper()', globals(), locals(), p)
profWrapper()
+class ScaleTimeRecord(object):
+ def __init__(self):
+ super(ScaleTimeRecord, self).__init__()
+
+
+
class TimerObject(object):
def __init__(self,name):
super(TimerObject, self).__init__()
@@ -1451,24 +1462,16 @@
self._queue = Queue.Queue()
#self.get = self._queue.get
- self.put_timer = TimerObject("enqueue")
- self.get_timer = TimerObject("dequeue")
- self.last_put_call = time.time()
- self.put_call_timer = TimerObject("put_delta")
+ #self.put_timer = TimerObject("enqueue")
+ #self.get_timer = TimerObject("dequeue")
+ #self.last_put_call = time.time()
+ #self.put_call_timer = TimerObject("put_delta")
def get(self):
- #before = time.time()
- #size = self._queue.qsize()
- while True:
- try:
- res = self._queue.get()
- break
- except:
- pass
- after = time.time()
- if res != None:
- self.get_timer.record(res[1], after, res[2])
- return res[0]
+ conn = self._queue.get()
+ if conn != None:
+ conn.scale_times.thread_after_get = datetime.utcnow()
+ return conn
def start(self):
"""Start the pool of threads."""
@@ -1485,16 +1488,14 @@
"""Number of worker threads which are idle. Read-only."""
return len([t for t in self._threads if t.conn is None])
idle = property(_get_idle, doc=_get_idle.__doc__)
+
+ def _get_busy(self):
+ """Number of worker threads which are busy. Read-only."""
+ return len([t for t in self._threads if not t.conn is None])
+ busy = property(_get_busy, doc=_get_idle.__doc__)
def put(self, obj):
- before = time.time()
- size = self._queue.qsize()
- self._queue.put((obj,before,size))
-
- self.put_call_timer.record(self.last_put_call, before, size)
- self.last_put_call = before
- #after = time.time()
- #self.put_timer.record(before, after, size)
+ self._queue.put(obj)
if obj is _SHUTDOWNREQUEST:
return
@@ -1561,9 +1562,9 @@
KeyboardInterrupt), exc1:
pass
- self.get_timer.dump()
- self.put_timer.dump()
- self.put_call_timer.dump()
+ #self.get_timer.dump()
+ #self.put_timer.dump()
+ #self.put_call_timer.dump()
@@ -1817,7 +1818,9 @@
def tick(self):
"""Accept a new connection and put it on the Queue."""
try:
+ before_accept = datetime.utcnow()
s, addr = self.socket.accept()
+ after_accept = datetime.utcnow()
if not self.ready:
return
@@ -1881,6 +1884,11 @@
makefile = self.ssl_adapter.makefile
conn = self.ConnectionClass(s, self.wsgi_app, environ, makefile)
+ conn.scale_times = ScaleTimeRecord()
+ conn.scale_times.tick_before_accept = before_accept
+ conn.scale_times.tick_after_accept = after_accept
+ conn.scale_times.tick_at_put = datetime.utcnow()
+ conn.scale_times.busy_threads = self.requests.busy
self.requests.put(conn)
except socket.timeout:
13 years, 3 months