moksha/widgets/metrics.py | 115 ++++++++++++++++++++++++++++++++-------------- pip.py | 12 ++++ stable-reqs.txt | 7 +- 3 files changed, 95 insertions(+), 39 deletions(-)
New commits: commit 6b54e9b1fa5d99eda99f33f239d9f3dabb363425 Author: Luke Macken lmacken@redhat.com Date: Thu Mar 12 18:35:47 2009 -0400
Update our stable-reqs
diff --git a/stable-reqs.txt b/stable-reqs.txt index 00ba062..e16d826 100644 --- a/stable-reqs.txt +++ b/stable-reqs.txt @@ -30,8 +30,7 @@ WebFlash==0.1a8dev1 WebHelpers==0.6.4 WebOb==0.9.6.1 WebTest==1.1 -## FIXME: could not find svn URL in dependency_links for this package: -##WidgetBrowser==0.1dev-20090311 +#WidgetBrowser==0.1 boto==1.6b decorator==3.0.0 demjson==1.4 @@ -40,7 +39,7 @@ feedcache==1.3 feedparser==4.1 lxml==2.2beta4 meld3==0.6.5 --e git+git+ssh://git.fedorahosted.org/git/moksha@f97231a017e5736005e50e94eea2c2b55a4d891a#egg=moksha-0.1dev-py2.5-f97231a017e5736005e50e94eea2c2b55a4d891a +-e git+git+ssh://git.fedorahosted.org/git/moksha@03438a82c9b26b700834e9e17d31ab8c6ace8d1a#egg=moksha-0.1dev-py2.5-dev morbid==0.8.4 nose==0.10.4 orbited==0.7.7
commit 03438a82c9b26b700834e9e17d31ab8c6ace8d1a Merge: d8ed9da... c826f94... Author: Luke Macken lmacken@redhat.com Date: Thu Mar 12 17:53:11 2009 -0400
Merge branch 'master' of git+ssh://git.fedorahosted.org/git/moksha
commit d8ed9da6aba820f111fd29df54d690edfbb3dee0 Author: Luke Macken lmacken@redhat.com Date: Thu Mar 12 17:52:40 2009 -0400
Get the Moksha metrics app working with multiple CPUs and multiprocess apps
diff --git a/moksha/widgets/metrics.py b/moksha/widgets/metrics.py index 2cfac24..a5ae062 100644 --- a/moksha/widgets/metrics.py +++ b/moksha/widgets/metrics.py @@ -167,24 +167,48 @@ class MokshaMessageMetricsWidget(LiveFlotWidget): super(MokshaMessageMetricsWidget, self).update_params(d)
+PID = 0 +NAME = -1 +MEM_TOTAL = -2 + class MokshaMetricsDataStream(PollingDataStream): - frequency = 2 - procs = ('orbited', 'paster', 'moksha') + frequency = 3 + procs = ('orbited', 'paster', 'moksha', 'httpd') cpu_usage = defaultdict(list) programs = None + pids = {} + count = 0 + + # Unless we are monitoring apache, only poll for pids once + poll_for_new_pids = False
def __init__(self): - self.programs = [] + self.programs = self._find_programs() + self.processors = self._find_processors() + super(MokshaMetricsDataStream, self).__init__() + + def _find_programs(self): + programs = [] for program in self.mem(): for proc in self.procs: - if program[-1].startswith(proc) or proc in program[-1]: - self.programs.append(program) - - super(MokshaMetricsDataStream, self).__init__() + if program[NAME].startswith(proc) or proc in program[NAME]: + programs.append(program) + for pid in program[PID].split(','): + self.pids[int(pid)] = program[NAME] + if proc == 'httpd': + self.poll_for_new_pids = True + return programs + + def _find_processors(self): + """ Find the number of processors """ + processors = 0 + for line in file('/proc/cpuinfo').readlines(): + if line.startswith('processor'): + processors = int(line.split(':')[1]) + return processors + 1
def poll(self): i = 0 - pids = {} mem_data = { 'data': [], 'options': { @@ -196,7 +220,7 @@ class MokshaMetricsDataStream(PollingDataStream): 'data': [], 'options': { 'xaxis': {'min': 0, 'max': 50}, - 'yaxis': {'min': 0, 'max': 100}, + 'yaxis': {'min': 0, 'max': 100 * self.processors}, 'legend': { 'position': 'nw', 'backgroundColor': 'null' @@ -204,48 +228,68 @@ class MokshaMetricsDataStream(PollingDataStream): } }
+ self.count += 1 + if self.poll_for_new_pids and self.count % 10 == 0: + self.programs = self._find_programs() + self.count = 0 + for program in self.programs: for proc in self.procs: - if program[-1].startswith(proc) or proc in program[-1]: - pids[program[0]] = program[-1] - y = float(program[-2].split()[0]) + if program[NAME].startswith(proc) or proc in program[NAME]: + total_mem_usage = float(program[MEM_TOTAL].split()[0]) mem_data['data'].append({ - 'data': [[i, y]], + 'data': [[i, total_mem_usage]], 'bars': {'show': 'true'}, - 'label': program[-1], + 'label': program[NAME], }) mem_data['options']['xaxis']['ticks'].append( - [i + 0.5, program[-1]]) + [i + 0.5, program[NAME]]) i += 1
self.send_message('moksha_mem_metrics', [mem_data])
- cmd = ['/usr/bin/top', '-b', '-n 1'] - for pid in pids: - cmd += ['-p %s' % pid] - - p = subprocess.Popen(cmd, stdout=subprocess.PIPE) - out, err = p.communicate() - out = out.strip().split('\n') - for i, line in enumerate(out): - if line.lstrip().startswith('PID'): - out = out[i+1:] - break - + # top only allows 20 pids to be specified at a time + # so make multiple calls if we have to + out = [] + _pids = [] + _pids.extend(self.pids.keys()) + while _pids: + cmd = ['/usr/bin/top', '-b', '-n 1'] + for pid in _pids[:20]: + cmd += ['-p %s' % pid] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + stdout, stderr = p.communicate() + stdout = stdout.strip().split('\n') + for i, line in enumerate(stdout): + if line.lstrip().startswith('PID'): + out += stdout[i+1:] + break + _pids = _pids[20:] + + # Determine the % CPU usage for each pid + pid_cpu = defaultdict(float) for line in out: splitline = line.split() - pid = splitline[0] + if not splitline: continue + pid = int(splitline[0]) cpu_usage = float(splitline[-4]) - for history in self.cpu_usage[pid]: + pid_cpu[self.pids[pid]] += cpu_usage + + # Move everything to the left, and append the latest CPU usage. + for program, cpu_usage in pid_cpu.items(): + for history in self.cpu_usage[program]: history[0] -= 1 - self.cpu_usage[pid].append([50, cpu_usage]) - self.cpu_usage[pid] = self.cpu_usage[pid][-51:]
- for pid, history in self.cpu_usage.items(): + self.cpu_usage[program].append([50, cpu_usage]) + + # Only store 50 ticks worth of data + self.cpu_usage[program] = self.cpu_usage[program][-51:] + + for program, history in self.cpu_usage.items(): cpu_data['data'].append({ 'data': history, 'lines': {'show': 'true', 'fill': 'true'}, - 'label': pids[pid], + 'label': program.split()[0], })
self.send_message('moksha_cpu_metrics', [cpu_data]) @@ -354,9 +398,10 @@ class MokshaMetricsDataStream(PollingDataStream): cmds[cmd]=cmds.setdefault(cmd,0)+private if count.has_key(cmd): count[cmd] += 1 + pids[cmd] += ',%d' % pid else: count[cmd] = 1 - pids[cmd] = pid + pids[cmd] = str(pid)
#Add shared mem for each program total=0 @@ -384,7 +429,7 @@ class MokshaMetricsDataStream(PollingDataStream):
for cmd in sort_list: results.append([ - "%d" % pids[cmd[0]], + "%s" % pids[cmd[0]], "%sB" % human(cmd[1]-shareds[cmd[0]]), "%sB" % human(shareds[cmd[0]]), "%sB" % human(cmd[1]),
commit 90edff9963c60423c9b227a50916c35f857faddc Author: Luke Macken lmacken@redhat.com Date: Wed Mar 11 20:44:23 2009 -0400
Update our stable version of Extremes
diff --git a/stable-reqs.txt b/stable-reqs.txt index c08bb49..00ba062 100644 --- a/stable-reqs.txt +++ b/stable-reqs.txt @@ -2,7 +2,7 @@ AddOns==0.6 Beaker==1.2.3 BytecodeAssembler==0.3 DecoratorTools==1.7 -Extremes==1.1 +Extremes==1.1.1 FormEncode==1.2.1 Genshi==0.5.1 Jinja==1.2 @@ -31,7 +31,7 @@ WebHelpers==0.6.4 WebOb==0.9.6.1 WebTest==1.1 ## FIXME: could not find svn URL in dependency_links for this package: -#WidgetBrowser==0.1dev-20090311 +##WidgetBrowser==0.1dev-20090311 boto==1.6b decorator==3.0.0 demjson==1.4
commit d25b7a6ec6448054ebae38c33ff6d4eb15e46a19 Author: Luke Macken lmacken@redhat.com Date: Wed Mar 11 20:42:32 2009 -0400
Add a Git.parse_checkout_text method to fill in some more of pip's git API
diff --git a/pip.py b/pip.py index 375aa96..172d96c 100755 --- a/pip.py +++ b/pip.py @@ -2765,6 +2765,18 @@ class Git(VersionControl): return url, rev return None, None
+ def parse_checkout_text(self, text): + url = None + rev = None + for line in text.splitlines(): + if not line.strip() or line.strip().startswith('#'): + continue + if line.startswith('git remote'): + url = line.split()[-2] + elif line.startswith('git checkout'): + rev = line.split()[-1] + return url, rev + def unpack(self, location): """Clone the Git repository at the url to the destination location""" url, rev = self.get_url_rev()
moksha-commits@lists.fedorahosted.org