modules/helpers/metrics-simulator/pom.xml | 2 modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/InsertionTimes.java | 60 ----- modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/MeasurementCollector.java | 3 modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/RawInserts.java | 92 -------- modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Simulator.java | 2 modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Stats.java | 109 ---------- modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/StatsCollector.java | 50 +++- modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/stats/Aggregate.java | 69 ++++++ modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/stats/Stats.java | 103 +++++++++ modules/helpers/metrics-simulator/src/main/scripts/assembly.xml | 2 10 files changed, 215 insertions(+), 277 deletions(-)
New commits: commit 07341c2c219e35d726b7a0ac1785615cbcc2b661 Author: John Sanda jsanda@redhat.com Date: Fri Mar 8 14:07:54 2013 -0500
cleaning up statistics sampling and reporting
diff --git a/modules/helpers/metrics-simulator/pom.xml b/modules/helpers/metrics-simulator/pom.xml index 7ad4b47..d5bf8ae 100644 --- a/modules/helpers/metrics-simulator/pom.xml +++ b/modules/helpers/metrics-simulator/pom.xml @@ -91,6 +91,8 @@ <artifactId>maven-assembly-plugin</artifactId> <version>2.4</version> <configuration> + <finalName>rhq-metrics-simulator</finalName> + <appendAssemblyId>false</appendAssemblyId> <descriptors> <descriptor>src/main/scripts/assembly.xml</descriptor> </descriptors> diff --git a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/InsertionTimes.java b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/InsertionTimes.java deleted file mode 100644 index 6c3c023..0000000 --- a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/InsertionTimes.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * - * * RHQ Management Platform - * * Copyright (C) 2005-2012 Red Hat, Inc. - * * All rights reserved. - * * - * * This program is free software; you can redistribute it and/or modify - * * it under the terms of the GNU General Public License, version 2, as - * * published by the Free Software Foundation, and/or the GNU Lesser - * * General Public License, version 2.1, also as published by the Free - * * Software Foundation. - * * - * * This program is distributed in the hope that it will be useful, - * * but WITHOUT ANY WARRANTY; without even the implied warranty of - * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * * GNU General Public License and the GNU Lesser General Public License - * * for more details. - * * - * * You should have received a copy of the GNU General Public License - * * and the GNU Lesser General Public License along with this program; - * * if not, write to the Free Software Foundation, Inc., - * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ - -package org.rhq.metrics.simulator; - -/** - * @author John Sanda - */ -public class InsertionTimes { - - private double max; - private double min; - private double mean; - private double standardDeviation; - - public InsertionTimes(double max, double min, double mean, double standardDeviation) { - this.max = max; - this.min = min; - this.mean = mean; - this.standardDeviation = standardDeviation; - } - - public double getMax() { - return max; - } - - public double getMin() { - return min; - } - - public double getMean() { - return mean; - } - - public double getStandardDeviation() { - return standardDeviation; - } -} diff --git a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/MeasurementCollector.java b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/MeasurementCollector.java index e265a18..35f575f 100644 --- a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/MeasurementCollector.java +++ b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/MeasurementCollector.java @@ -34,6 +34,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory;
import org.rhq.core.domain.measurement.MeasurementDataNumeric; +import org.rhq.metrics.simulator.stats.Stats; import org.rhq.server.metrics.MetricsServer;
/** @@ -130,7 +131,7 @@ public class MeasurementCollector implements Runnable { } finally { long endTime = System.currentTimeMillis(); long totalTime = endTime - startTime; - stats.addRawDataInsertTime(totalTime); + stats.addRawInsertTime(totalTime); log.info("Finished collecting and storing " + metricsCollected + " raw metric in " +totalTime + " ms."); } } diff --git a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/RawInserts.java b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/RawInserts.java deleted file mode 100644 index 60b3d59..0000000 --- a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/RawInserts.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * - * * RHQ Management Platform - * * Copyright (C) 2005-2012 Red Hat, Inc. - * * All rights reserved. - * * - * * This program is free software; you can redistribute it and/or modify - * * it under the terms of the GNU General Public License, version 2, as - * * published by the Free Software Foundation, and/or the GNU Lesser - * * General Public License, version 2.1, also as published by the Free - * * Software Foundation. - * * - * * This program is distributed in the hope that it will be useful, - * * but WITHOUT ANY WARRANTY; without even the implied warranty of - * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * * GNU General Public License and the GNU Lesser General Public License - * * for more details. - * * - * * You should have received a copy of the GNU General Public License - * * and the GNU Lesser General Public License along with this program; - * * if not, write to the Free Software Foundation, Inc., - * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ - -package org.rhq.metrics.simulator; - -/** - * @author John Sanda - */ -public class RawInserts implements Comparable<RawInserts> { - - private long timestamp; - - private int count; - - public RawInserts(long timestamp, int count) { - this.timestamp = timestamp; - this.count = count; - } - - public long getTimestamp() { - return timestamp; - } - - public int getCount() { - return count; - } - - @Override - public int compareTo(RawInserts other) { - if (timestamp < other.timestamp) { - return -1; - } - - if (timestamp > other.timestamp) { - return 1; - } - - if (count < other.count) { - return -1; - } - - if (count > other.count) { - return 1; - } - - return 0; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - RawInserts that = (RawInserts) o; - - if (timestamp != that.timestamp) return false; - if (count != that.count) return false; - - return true; - } - - @Override - public int hashCode() { - int result = (int) (timestamp ^ (timestamp >>> 32)); - result = 31 * result + count; - return result; - } - - -} diff --git a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Simulator.java b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Simulator.java index ee72d4c..8901538 100644 --- a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Simulator.java +++ b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Simulator.java @@ -56,6 +56,7 @@ import org.rhq.cassandra.schema.SchemaManager; import org.rhq.metrics.simulator.plan.ClusterConfig; import org.rhq.metrics.simulator.plan.ScheduleGroup; import org.rhq.metrics.simulator.plan.SimulationPlan; +import org.rhq.metrics.simulator.stats.Stats; import org.rhq.server.metrics.DateTimeService; import org.rhq.server.metrics.MetricsServer;
@@ -123,6 +124,7 @@ public class Simulator implements ShutdownManager { Thread.sleep(Minutes.minutes(plan.getSimulationTime()).toStandardDuration().getMillis()); } catch (InterruptedException e) { } + statsCollector.reportSummaryStats(); log.info("Simulation has completed. Initiating shutdown..."); shutdown(0); } diff --git a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Stats.java b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Stats.java deleted file mode 100644 index bf61369..0000000 --- a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/Stats.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * - * * RHQ Management Platform - * * Copyright (C) 2005-2012 Red Hat, Inc. - * * All rights reserved. - * * - * * This program is free software; you can redistribute it and/or modify - * * it under the terms of the GNU General Public License, version 2, as - * * published by the Free Software Foundation, and/or the GNU Lesser - * * General Public License, version 2.1, also as published by the Free - * * Software Foundation. - * * - * * This program is distributed in the hope that it will be useful, - * * but WITHOUT ANY WARRANTY; without even the implied warranty of - * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * * GNU General Public License and the GNU Lesser General Public License - * * for more details. - * * - * * You should have received a copy of the GNU General Public License - * * and the GNU Lesser General Public License along with this program; - * * if not, write to the Free Software Foundation, Inc., - * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ - -package org.rhq.metrics.simulator; - -import java.util.NavigableSet; -import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; - -/** - * @author John Sanda - */ -public class Stats { - - /** - * The total number of raw data inserted - */ - private AtomicLong totalRawInserts = new AtomicLong(0); - - /** - * A set of per-minute samples of raw inserts - */ - private NavigableSet<RawInserts> rawInsertsPerMinute = new TreeSet<RawInserts>(); - - private int rawInsertsThisInterval; - - private long currentInterval; - - private DescriptiveStatistics rawDataInsertTimes = new DescriptiveStatistics(200); - - private ReentrantLock insertCountsLock = new ReentrantLock(); - - private ReentrantLock insertTimesLock = new ReentrantLock(); - - public void addRawInserts(int count) { - totalRawInserts.addAndGet(count); - try { - insertCountsLock.lock(); - rawInsertsThisInterval += count; - } finally { - insertCountsLock.unlock(); - } - } - - public long getTotalRawInserts() { - return totalRawInserts.get(); - } - - public RawInserts getRawInsertsForLastInterval() { - return rawInsertsPerMinute.pollLast(); - } - - public void startNewInterval(long startTime) { - int insertsLastInterval; - try { - insertCountsLock.lock(); - insertsLastInterval = rawInsertsThisInterval; - rawInsertsThisInterval = 0; - } finally { - insertCountsLock.unlock(); - } - rawInsertsPerMinute.add(new RawInserts(currentInterval, insertsLastInterval)); - currentInterval = startTime; - } - - public void addRawDataInsertTime(long time) { - try { - insertTimesLock.lock(); - rawDataInsertTimes.addValue(time); - } finally { - insertTimesLock.unlock(); - } - } - - public InsertionTimes getInsertionTimes() { - try { - insertTimesLock.lock(); - return new InsertionTimes(rawDataInsertTimes.getMax(), rawDataInsertTimes.getMin(), - rawDataInsertTimes.getMean(), rawDataInsertTimes.getStandardDeviation()); - } finally { - insertTimesLock.unlock(); - } - } -} diff --git a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/StatsCollector.java b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/StatsCollector.java index 7f2c35e..b59ca9c 100644 --- a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/StatsCollector.java +++ b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/StatsCollector.java @@ -25,10 +25,14 @@
package org.rhq.metrics.simulator;
+import java.text.SimpleDateFormat; import java.util.Date;
import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.joda.time.Duration; + +import org.rhq.metrics.simulator.stats.Stats;
/** * @author John Sanda @@ -39,34 +43,52 @@ public class StatsCollector implements Runnable {
private Stats stats;
+ private long previousRawInsertTotal; + + private long lastRunTimestamp; + + private SimpleDateFormat dateFormat; + public StatsCollector(Stats stats) { this.stats = stats; + dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); }
@Override public void run() { - long totalRawInserts = stats.getTotalRawInserts(); long now = System.currentTimeMillis(); - stats.startNewInterval(now); - RawInserts inserts = stats.getRawInsertsForLastInterval(); + long totalRawInserts = stats.getTotalRawInserts();
// inserts will be null on the first run - if (inserts == null) { + if (lastRunTimestamp == 0) { + lastRunTimestamp = now; + previousRawInsertTotal = totalRawInserts; return; }
- log.info("Inserted " + totalRawInserts + " raw metrics in total."); - log.info("Inserted " + inserts.getCount() + " for interval starting at " + new Date(inserts.getTimestamp())); - logInsertionTimeStats(); + long lastRawInsertsCount = totalRawInserts - previousRawInsertTotal; + Duration duration = new Duration(lastRunTimestamp, now); + stats.addRawInsertsPerMinute(lastRawInsertsCount); + + StringBuilder data = new StringBuilder("Statistics Report\n") + .append("------------------------------------------------------------------------------------\n") + .append("Sampling period start time: " + dateFormat.format(new Date(lastRunTimestamp))).append("\n") + .append("Sampling period length: " + duration.toStandardSeconds().getSeconds()).append(" seconds\n") + .append("Total raw metrics inserted: ").append(totalRawInserts).append("\n") + .append("Raw inserts this sampling period: ").append(lastRawInsertsCount).append("\n") + .append(stats.getRawInsertsPerMinute()).append("\n") + .append(stats.getRawInsertTimes()).append("\n") + .append("------------------------------------------------------------------------------------"); + + log.info(data); + + lastRunTimestamp = now; + previousRawInsertTotal = totalRawInserts; }
- private void logInsertionTimeStats() { - InsertionTimes insertionTimes = stats.getInsertionTimes(); - log.info("Summary of raw insertion time stats (milliseconds):\n" + - "Min: " + insertionTimes.getMin() + "\n" + - "Mean: " + insertionTimes.getMean() + "\n" + - "Max: " + insertionTimes.getMax() + "\n" + - "Standard Deviation: " + insertionTimes.getStandardDeviation()); + public void reportSummaryStats() { + log.info("Reporting statistics for entire simulation run."); + log.info(stats.getRawInsertsPerMinute()); }
} diff --git a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/stats/Aggregate.java b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/stats/Aggregate.java new file mode 100644 index 0000000..61a6b51 --- /dev/null +++ b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/stats/Aggregate.java @@ -0,0 +1,69 @@ +/* + * + * * RHQ Management Platform + * * Copyright (C) 2005-2012 Red Hat, Inc. + * * All rights reserved. + * * + * * This program is free software; you can redistribute it and/or modify + * * it under the terms of the GNU General Public License, version 2, as + * * published by the Free Software Foundation, and/or the GNU Lesser + * * General Public License, version 2.1, also as published by the Free + * * Software Foundation. + * * + * * This program is distributed in the hope that it will be useful, + * * but WITHOUT ANY WARRANTY; without even the implied warranty of + * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * * GNU General Public License and the GNU Lesser General Public License + * * for more details. + * * + * * You should have received a copy of the GNU General Public License + * * and the GNU Lesser General Public License along with this program; + * * if not, write to the Free Software Foundation, Inc., + * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +package org.rhq.metrics.simulator.stats; + +/** + * @author John Sanda + */ +public class Aggregate { + + private String name; + private double max; + private double min; + private double mean; + private double standardDeviation; + + + public Aggregate(String name, double max, double min, double mean, double standardDeviation) { + this.name = name; + this.max = max; + this.min = min; + this.mean = mean; + this.standardDeviation = standardDeviation; + } + + public double getMax() { + return max; + } + + public double getMin() { + return min; + } + + public double getMean() { + return mean; + } + + public double getStandardDeviation() { + return standardDeviation; + } + + @Override + public String toString() { + return name + ": {min: " + getMin() + ", mean: " + getMean() + ", max: " + getMax() + ", standardDeviation: " + + getStandardDeviation() + "}"; + } +} diff --git a/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/stats/Stats.java b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/stats/Stats.java new file mode 100644 index 0000000..604f1ba --- /dev/null +++ b/modules/helpers/metrics-simulator/src/main/java/org/rhq/metrics/simulator/stats/Stats.java @@ -0,0 +1,103 @@ +/* + * + * * RHQ Management Platform + * * Copyright (C) 2005-2012 Red Hat, Inc. + * * All rights reserved. + * * + * * This program is free software; you can redistribute it and/or modify + * * it under the terms of the GNU General Public License, version 2, as + * * published by the Free Software Foundation, and/or the GNU Lesser + * * General Public License, version 2.1, also as published by the Free + * * Software Foundation. + * * + * * This program is distributed in the hope that it will be useful, + * * but WITHOUT ANY WARRANTY; without even the implied warranty of + * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * * GNU General Public License and the GNU Lesser General Public License + * * for more details. + * * + * * You should have received a copy of the GNU General Public License + * * and the GNU Lesser General Public License along with this program; + * * if not, write to the Free Software Foundation, Inc., + * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +package org.rhq.metrics.simulator.stats; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; + +/** + * @author John Sanda + */ +public class Stats { + + /** + * The total number of raw inserts + */ + private AtomicLong totalRawInserts = new AtomicLong(0); + + private AtomicInteger rawInsertsThisMinute = new AtomicInteger(0); + + private DescriptiveStatistics rawInsertsPerMinute = new DescriptiveStatistics(200); + + private DescriptiveStatistics rawInsertTimesPerMinute = new DescriptiveStatistics(200); + + private ReentrantLock insertTimesLock = new ReentrantLock(); + + public void addRawInserts(int count) { + totalRawInserts.addAndGet(count); + rawInsertsThisMinute.addAndGet(count); + } + + public long getTotalRawInserts() { + return totalRawInserts.get(); + } + + /** + * Called by measurement collectors to report insertion times. This method uses an + * internal lock to allow for concurrent access. + * + * @param time The time to insert a set of raw metrics + */ + public void addRawInsertTime(long time) { + try { + insertTimesLock.lock(); + rawInsertTimesPerMinute.addValue(time); + } finally { + insertTimesLock.unlock(); + } + } + + public Aggregate getRawInsertTimes() { + try { + insertTimesLock.lock(); + return new Aggregate("raw insertion times (milliseconds)", rawInsertTimesPerMinute.getMax(), + rawInsertTimesPerMinute.getMin(), rawInsertTimesPerMinute.getMean(), + rawInsertTimesPerMinute.getStandardDeviation()); + } finally { + insertTimesLock.unlock(); + } + } + + /** + * Called by {@link org.rhq.metrics.simulator.StatsCollector} to report the number of raw inserts for a given + * minute. Since there is only a single {@link org.rhq.metrics.simulator.StatsCollector} this method does not + * support concurrent access. + * + * @param value The number of raw metrics inserted in a given minute. + */ + public void addRawInsertsPerMinute(long value) { + rawInsertsPerMinute.addValue(value); + } + + public Aggregate getRawInsertsPerMinute() { + return new Aggregate("Raw inserts per minute", rawInsertsPerMinute.getMax(), rawInsertsPerMinute.getMin(), + rawInsertsPerMinute.getMean(), rawInsertsPerMinute.getStandardDeviation()); + } + +} diff --git a/modules/helpers/metrics-simulator/src/main/scripts/assembly.xml b/modules/helpers/metrics-simulator/src/main/scripts/assembly.xml index 2474a01..0b081c0 100644 --- a/modules/helpers/metrics-simulator/src/main/scripts/assembly.xml +++ b/modules/helpers/metrics-simulator/src/main/scripts/assembly.xml @@ -9,7 +9,7 @@ <format>zip</format> </formats> <includeBaseDirectory>true</includeBaseDirectory> - <baseDirectory>rhq-metrics-simulator-${project.version}</baseDirectory> <!-- matches the zip when it is renamed in rhq-container.build.xml --> + <baseDirectory>rhq-metrics-simulator-${project.version}</baseDirectory>
<fileSets>
rhq-commits@lists.fedorahosted.org