Author: tmckay Date: 2011-01-26 19:35:08 +0000 (Wed, 26 Jan 2011) New Revision: 4486
Modified: branches/scale_testing/mech_scripts/test_driver.py Log: Record custom headers for offline processing
Modified: branches/scale_testing/mech_scripts/test_driver.py =================================================================== --- branches/scale_testing/mech_scripts/test_driver.py 2011-01-24 19:06:57 UTC (rev 4485) +++ branches/scale_testing/mech_scripts/test_driver.py 2011-01-26 19:35:08 UTC (rev 4486) @@ -33,7 +33,7 @@ self.visited = {}
self.file_init = False - self.file_number = -1 + self.file_number = 0 self.file_iterations = 0
def next_file(self): @@ -46,43 +46,67 @@ else: self.file_list = [self.link_file]
- self.file_number += 1 - if self.file_number == len(self.file_list): + name = self.file_list[self.file_number] + if self.file_number == len(self.file_list) - 1: self.file_iterations += 1 self.file_number = 0 - return self.file_list[self.file_number] + else: + self.file_number += 1 + return name
+ def handle_custom_headers(self, curr_file, link, resp, insert_blank): + fname = self.group + self.thread + "_" + os.path.basename(curr_file) + if os.path.isfile(fname): + f = open(fname, "a") + if insert_blank: + f.write("\n") + else: + f = open(fname, "w") + + f.write(link) + for i in ("page_before_accept", "page_after_accept", "page_put_time", + "page_after_get", "page_begin_service", "page_begin_proc", + "page_end_proc","page_send_headers"): + f.write(","+resp.getheader(i,"0")) + f.write("\n") + f.close() + def run(self): try: - # parse options on first invocation, get machine and links file + # use this to get an idea of tets overhead. + # Sum the times for individual gets to look just at page requests + start_timer = time_time() + + # parse options on first invocation if self.machine == None: parser = optparse.OptionParser() parser.add_option('--machine', dest='machine', type='str', default="localhost:45672") - parser.add_option('--file', dest='file', type='str', default="") - parser.add_option('--iter', dest='iter', type='int', default=0) + parser.add_option('--file', dest='file', type='str', default="./links") + parser.add_option('--iter', dest='iter', type='int', default=1) parser.add_option('--info', dest='info', action="store_true", default=False) + + # combine these to create an output file name for web server internal stats + parser.add_option('--group', dest='group', type='str', default="headers") + parser.add_option('--thread', dest='thread', type='str', default="") + if type(self.script_options) == str: self.script_options = self.script_options.split() opts, args = parser.parse_args(self.script_options) + self.machine = opts.machine self.link_file = os.path.abspath(opts.file) self.iteration_limit = opts.iter self.print_info = opts.info - - start_timer = time_time() - f = open(self.next_file(), "r") + self.group = opts.group + self.thread = opts.thread + + curr_file = self.next_file() + f = open(curr_file, "r") if self.print_info: - print f + print curr_file
- # If an iteration limit was set, bail after going through file list - # that many times - if self.iteration_limit and self.file_iterations == self.iteration_limit: - return True - - total_gets = 0 - total_size = 0 - get_latency = 0 - proc_time = 0 + total_gets = total_size = get_latency = 0 + first_link = True for link in f: link_info = link.split(',')
@@ -104,6 +128,11 @@ conn.request('GET', link) resp = conn.getresponse() data = resp.read() + get_latency += time_time() - time_for_get + total_gets += 1 + total_size += len(data) + + # Handle response, update visited times if necessary if resp.status == 200: self.visited[link] = datetime.utcnow().strftime(self.http_date_gmt) if self.print_info: @@ -115,26 +144,22 @@ self.visited[link] = None if self.print_info: print ("other status", resp.status, link) - get_latency += time_time() - time_for_get - total_gets += 1 - total_size += len(data)
- # get processing time header - proc_time_header = resp.getheader("page_request_time",None) - if proc_time_header != None: - tmp = dateparser.parse(proc_time_header) - proc_time += tmp.second + tmp.microsecond / 1000000.0 - if self.print_info: - print proc_time_header + # write custom headers to a file -- too much overhead to + # embed in the test. parse(datetime) is apparently expensive + self.handle_custom_headers(curr_file, link, resp, first_link) + first_link = False
- latency = time_time() - start_timer - self.custom_timers['Page File'] = latency + # Record some custom stuff + self.custom_timers['Page File'] = get_latency self.custom_timers['Avg Get'] = get_latency / total_gets self.stand_alone_stats['Total Gets'] = total_gets self.stand_alone_stats['Total Size kb'] = total_size / 1024.0 - self.stand_alone_stats['Page File Proc'] = proc_time - self.stand_alone_stats['Get Proc'] = proc_time / total_gets
+ # Get run time for entire test + latency = time_time() - start_timer + self.custom_timers['Total Test'] = latency + except Exception, e: traceback.print_exc()
@@ -142,26 +167,32 @@ trans = Transaction() trans.script_options=sys.argv[1:] avg_pg_file = 0 - avg_pg_proc = 0 - avg_get_proc = 0 avg_get = 0 avg_num_get = 0 avg_kb =0 + avg_total_test = 0 samples = 0
- while not trans.run(): + while True: + trans.run() samples += 1 avg_pg_file += trans.custom_timers['Page File'] - avg_pg_proc += trans.stand_alone_stats['Page File Proc'] - avg_get_proc += trans.stand_alone_stats['Get Proc'] avg_get += trans.custom_timers['Avg Get'] + avg_total_test += trans.custom_timers['Total Test'] avg_num_get += trans.stand_alone_stats['Total Gets'] avg_kb += trans.stand_alone_stats['Total Size kb'] + + # If an iteration limit was set, bail after going through file list + # that many times + if trans.iteration_limit and trans.file_iterations >= trans.iteration_limit: + break
- print "page: ", avg_pg_file / samples, \ - "page proc: ", avg_pg_proc / samples, \ - "get proc: ", avg_get_proc / samples, \ - "get:", avg_get / samples, "num_get:", avg_num_get /samples, "kb:", avg_kb / samples + print "avg page: ", avg_pg_file / samples, \ + "\navg get:", avg_get / samples, \ + "\navg num_get:", avg_num_get /samples, \ + "\navg page kb:", avg_kb / samples, \ + "\navg test:", avg_total_test / samples + #print trans.custom_timers #print trans.stand_alone_stats
cumin-developers@lists.fedorahosted.org