Author: tmckay
Date: 2011-02-09 18:45:32 +0000 (Wed, 09 Feb 2011)
New Revision: 4512
Modified:
branches/scale_testing/mech_scripts/process_times.py
branches/scale_testing/mech_scripts/test_driver.py
Log:
Handle blank lines in link files, sort pages by longest time in process_times.py
Modified: branches/scale_testing/mech_scripts/process_times.py
===================================================================
--- branches/scale_testing/mech_scripts/process_times.py 2011-02-09 14:41:07 UTC (rev
4511)
+++ branches/scale_testing/mech_scripts/process_times.py 2011-02-09 18:45:32 UTC (rev
4512)
@@ -9,6 +9,7 @@
import os
import csv
import optparse
+from operator import itemgetter
def dump(stats):
try:
@@ -80,11 +81,12 @@
f = open(p, "r")
for line in f:
line_count += 1
+ line = line.strip()
# Look for page end marker, will contain total size for that page for new stuff.
- # Old stuff might still have a blank line
- times = line.strip().split(",")
- if times[0] == "page_break" or times[0] == "":
+ # Old stuff might still have a blank line
+ times = line.split(",")
+ if times[0] == "page_break" or len(line) == 0:
last_page_size = page_size
if len(times) > 1:
page_size = int(times[1])
@@ -170,28 +172,43 @@
parser = optparse.OptionParser()
parser.add_option("--csvfile", dest="csvfile",
default="")
parser.add_option("--threads", dest="threads", type=int,
default=0)
+ parser.add_option("--sort", dest="sort",
action="store_true", default=False)
opts, args = parser.parse_args()
- if len(sys.argv) < 2:
+ if len(args) < 1:
print "Forgot the file, eh?"
return
# Allowing processing of a single page file, possibly with multiple iterations
# or an entire directory of page files, each with multiple iterations
file_list = []
- if os.path.isdir(sys.argv[1]):
- for i in os.listdir(sys.argv[1]):
- file_list.append(os.path.join(sys.argv[1], i))
+ if os.path.isdir(args[0]):
+ for i in os.listdir(args[0]):
+ file_list.append(os.path.join(args[0], i))
else:
- file_list = [sys.argv[1]]
+ file_list = [args[0]]
+ # Process all pages, keep a running list of segment totals
+ # Optionally set up for sorting by longest processing time
totals = get_zero_list()
- for i in file_list:
- page_values = process_file(i)
+ sorted_times = dict()
+ for f in file_list:
+ page_values = process_file(f)
for i in range(len(page_values)):
totals[i] += page_values[i]
+ if opts.sort:
+ sorted_times[f] = page_values
+ # element 5 will be the total for each page
+ if opts.sort:
+ def get(it):
+ return it[1][5]
+
+ a = sorted(sorted_times.items(), key=get)
+ for i in a:
+ print i[0], i[1][5]
+
# Find average of all pages processed over multiple iterations.
for i in range(len(totals)):
totals[i] = totals[i] / len(file_list)
Modified: branches/scale_testing/mech_scripts/test_driver.py
===================================================================
--- branches/scale_testing/mech_scripts/test_driver.py 2011-02-09 14:41:07 UTC (rev 4511)
+++ branches/scale_testing/mech_scripts/test_driver.py 2011-02-09 18:45:32 UTC (rev 4512)
@@ -35,6 +35,41 @@
self.file_iterations = 0
self.last_page_total = 0
+ def init(self):
+ parser = optparse.OptionParser()
+ parser.add_option('--machine', dest='machine',
type='str', default="localhost:45672")
+ parser.add_option('--file', dest='file', type='str',
default="./links")
+ parser.add_option('--iter', dest='iter', type='int',
default=1)
+ parser.add_option('--info', dest='info',
action="store_true", default=False)
+ parser.add_option('--noskip', dest='skip',
action="store_false", default=True)
+
+ # combine these to create an output file name for web server internal stats
+ parser.add_option('--group', dest='group', type='str',
default="headers")
+ parser.add_option('--thread', dest='thread', type='str',
default="")
+ parser.add_option('--outdir', dest='outdir', type='str',
default=".")
+
+
+ if type(self.script_options) == str:
+ self.script_options = self.script_options.split()
+
+ opts, args = parser.parse_args(self.script_options)
+
+ self.machine = opts.machine
+ self.link_file = os.path.abspath(opts.file)
+ self.iteration_limit = opts.iter
+ self.print_info = opts.info
+ self.skip = opts.skip
+ self.group = opts.group
+ self.thread = opts.thread
+ self.outdir = opts.outdir
+ if self.outdir != "." and not os.path.isdir(self.outdir):
+ try:
+ # in a multi thread environment, we may
+ # actually collide here in the creation.
+ os.mkdir(self.outdir)
+ except:
+ pass
+
def next_file(self):
if not self.file_init:
self.file_init = True
@@ -80,42 +115,10 @@
# Sum the times for individual gets to look just at page requests
start_timer = time_time()
- # parse options on first invocation
+ # set up on first invocation. It's done this way rather than in __init
+ # because of the way internals in multi-mechanize work.
if self.machine == None:
-
- parser = optparse.OptionParser()
- parser.add_option('--machine', dest='machine',
type='str', default="localhost:45672")
- parser.add_option('--file', dest='file',
type='str', default="./links")
- parser.add_option('--iter', dest='iter',
type='int', default=1)
- parser.add_option('--info', dest='info',
action="store_true", default=False)
- parser.add_option('--noskip', dest='skip',
action="store_false", default=True)
-
- # combine these to create an output file name for web server internal
stats
- parser.add_option('--group', dest='group',
type='str', default="headers")
- parser.add_option('--thread', dest='thread',
type='str', default="")
- parser.add_option('--outdir', dest='outdir',
type='str', default=".")
-
-
- if type(self.script_options) == str:
- self.script_options = self.script_options.split()
-
- opts, args = parser.parse_args(self.script_options)
-
- self.machine = opts.machine
- self.link_file = os.path.abspath(opts.file)
- self.iteration_limit = opts.iter
- self.print_info = opts.info
- self.skip = opts.skip
- self.group = opts.group
- self.thread = opts.thread
- self.outdir = opts.outdir
- if self.outdir != "." and not os.path.isdir(self.outdir):
- try:
- # in a multi thread environment, we may
- # actually collide here in the creation.
- os.mkdir(self.outdir)
- except:
- pass
+ self.init()
curr_file = self.next_file()
f = open(curr_file, "r")
@@ -125,8 +128,12 @@
total_gets = total_size = get_latency = 0
for link in f:
+ # skip blank lines
+ link = link.strip()
+ if len(link) == 0:
+ continue
link_info = link.split(',')
-
+
# Check for initial Boolean, indicates whether "If_Modified"
header was used
if len(link_info) == 3:
# Newest file type. If_Modified header, then Cache-Control