diff --git a/src/main/java/HdrToCsv.java b/src/main/java/HdrToCsv.java index 9b77399..5a19293 100644 --- a/src/main/java/HdrToCsv.java +++ b/src/main/java/HdrToCsv.java @@ -1,11 +1,8 @@ -import org.HdrHistogram.Histogram; import org.kohsuke.args4j.Option; -import psy.lob.saw.OrderedHistogramLogReader; +import psy.lob.saw.CsvConverter; -import java.io.File; -import java.io.FileNotFoundException; +import java.io.*; import java.nio.file.Paths; -import java.util.Locale; public class HdrToCsv implements Runnable { @@ -34,34 +31,13 @@ public void setInputFile(String fileName) @Override public void run() { - OrderedHistogramLogReader reader = null; - try + try(InputStream inputStream = new FileInputStream(inputFile)) { - reader = new OrderedHistogramLogReader(inputFile); + new CsvConverter(inputStream, System.out).convert(); } - catch (FileNotFoundException e) + catch (IOException e) { throw new RuntimeException(e); } - - System.out.println( - "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); - while (reader.hasNext()) - { - Histogram interval = (Histogram) reader.nextIntervalHistogram(); - System.out.printf(Locale.US, - "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", - interval.getStartTimeStamp() / 1000.0, - interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), - interval.getTotalCount(), interval.getMinValue(), - (long) interval.getMean(), - interval.getValueAtPercentile(50), - interval.getValueAtPercentile(90), - interval.getValueAtPercentile(95), - interval.getValueAtPercentile(99), - interval.getValueAtPercentile(99.9), - interval.getValueAtPercentile(99.99), - interval.getMaxValue()); - } } } \ No newline at end of file diff --git a/src/main/java/SplitHistogramLogs.java b/src/main/java/SplitHistogramLogs.java index b7b1e8c..d3adf06 100644 --- a/src/main/java/SplitHistogramLogs.java +++ b/src/main/java/SplitHistogramLogs.java @@ -1,14 +1,11 @@ -import org.HdrHistogram.Histogram; -import org.HdrHistogram.HistogramLogWriter; import org.kohsuke.args4j.Option; -import psy.lob.saw.HdrHistogramUtil; -import psy.lob.saw.OrderedHistogramLogReader; +import psy.lob.saw.HistogramsSplitter; import java.io.File; -import java.io.FileNotFoundException; -import java.util.HashMap; +import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.file.Path; import java.util.HashSet; -import java.util.Map; import java.util.Set; import static psy.lob.saw.HdrHistogramUtil.logHistogramForVerbose; @@ -27,6 +24,7 @@ public class SplitHistogramLogs implements Runnable private File inputFile; private Set excludeTags = new HashSet<>(); private Set includeTags = new HashSet<>(); + private String fileName; public static void main(String[] args) throws Exception { @@ -91,9 +89,11 @@ public void run() System.out.printf("start:%.2f end: MAX path:%s file:%s \n", start, absolutePath, name); } } - try + try(InputStream inputStream = new FileInputStream(inputFile)) { - split(); + Path outputDir = inputFile.toPath().getParent(); + fileName = inputFile.getName(); + new HistogramsSplitter(fileName, inputStream, start, end, verbose, this::shouldSkipTag, outputDir).split(); } catch (Exception e) { @@ -101,51 +101,9 @@ public void run() } } - private void split() throws FileNotFoundException - { - OrderedHistogramLogReader reader = new OrderedHistogramLogReader( - inputFile, - start, - end, - tag -> shouldSkipTag(tag)); - Map writerByTag = new HashMap<>(); - Histogram interval; - int i = 0; - while (reader.hasNext()) - { - interval = (Histogram) reader.nextIntervalHistogram(); - if (interval == null) - { - continue; - } - String ntag = interval.getTag(); - if (shouldSkipTag(ntag)) - { - throw new IllegalStateException("Should be filtered upfront by the reader"); - } - if (verbose) - { - logHistogramForVerbose(System.out, interval, i++); - } - interval.setTag(null); - HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k)); - writer.outputIntervalHistogram(interval); - - } - } - private boolean shouldSkipTag(String ntag) { ntag = (ntag == null) ? "default" : ntag; return excludeTags.contains(ntag) || (!includeTags.isEmpty() && !includeTags.contains(ntag)); } - - private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag) - { - tag = (tag == null) ? "default" : tag; - File outputFile = new File(tag + "." + inputFile.getName()); - String comment = "Splitting of:" + inputFile.getName() + " start:" + start + " end:" + end; - HistogramLogWriter writer = HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); - return writer; - } -} +} \ No newline at end of file diff --git a/src/main/java/SummarizeHistogramLogs.java b/src/main/java/SummarizeHistogramLogs.java index a62f9c7..97b9bad 100644 --- a/src/main/java/SummarizeHistogramLogs.java +++ b/src/main/java/SummarizeHistogramLogs.java @@ -3,10 +3,7 @@ import org.kohsuke.args4j.Option; import psy.lob.saw.OrderedHistogramLogReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.PrintStream; +import java.io.*; import java.util.*; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -134,69 +131,77 @@ private void summarizeAndPrint() throws FileNotFoundException long intervalLengthSum = 0; for (File inputFile : inputFiles) { - if (verbose) + try(InputStream inputStream = new FileInputStream(inputFile)) { - System.out.println("Summarizing file: " + inputFile.getName()); - } - OrderedHistogramLogReader reader = new OrderedHistogramLogReader( - inputFile, - start, - end, - tag -> shouldSkipTag(tag)); - Histogram interval; - int i = 0; - boolean first = true; - long startTime = 0; - - while (reader.hasNext()) - { - interval = (Histogram) reader.nextIntervalHistogram(); - if (interval == null) + if (verbose) { - continue; + System.out.println("Summarizing file: " + inputFile.getName()); } - if (first) + OrderedHistogramLogReader reader = new OrderedHistogramLogReader( + inputStream, + start, + end, + tag -> shouldSkipTag(tag)); + Histogram interval; + int i = 0; + boolean first = true; + long startTime = 0; + + while (reader.hasNext()) { - first = false; - startTime = interval.getStartTimeStamp(); - if (verbose) + interval = (Histogram) reader.nextIntervalHistogram(); + if (interval == null) { - System.out.println("StartTime: " + new Date(startTime)); + continue; } + if (first) + { + first = false; + startTime = interval.getStartTimeStamp(); + if (verbose) + { + System.out.println("StartTime: " + new Date(startTime)); + } + } + String ntag = ignoreTag ? null : interval.getTag(); + final int numberOfSignificantValueDigits = interval.getNumberOfSignificantValueDigits(); + Histogram sum = sumByTag.computeIfAbsent(ntag, k -> + { + Histogram h = new Histogram(numberOfSignificantValueDigits); + h.setTag(k); + return h; + }); + final long intervalLength = interval.getEndTimeStamp() - interval.getStartTimeStamp(); + intervalLengthSum += intervalLength; + sum.add(interval); + if (verbose) + { + logHistogramForVerbose(System.out, interval, i++, outputValueUnitRatio); + } } - String ntag = ignoreTag ? null : interval.getTag(); - final int numberOfSignificantValueDigits = interval.getNumberOfSignificantValueDigits(); - Histogram sum = sumByTag.computeIfAbsent(ntag, k -> - { - Histogram h = new Histogram(numberOfSignificantValueDigits); - h.setTag(k); - return h; - }); - final long intervalLength = interval.getEndTimeStamp() - interval.getStartTimeStamp(); - intervalLengthSum += intervalLength; - sum.add(interval); - if (verbose) + // calculate period + long maxPeriod = 0; + for (Histogram sum : sumByTag.values()) { - logHistogramForVerbose(System.out, interval, i++, outputValueUnitRatio); + long sumPeriod = (sum.getEndTimeStamp() - sum.getStartTimeStamp()); + if (verbose) + { + System.out.print(inputFile.getName()); + System.out.print(", "); + logHistogramForVerbose(System.out, sum, i++, outputValueUnitRatio); + } + sum.setEndTimeStamp(0); + sum.setStartTimeStamp(Long.MAX_VALUE); + maxPeriod = Math.max(maxPeriod, sumPeriod); } + period += maxPeriod; } - // calculate period - long maxPeriod = 0; - for (Histogram sum : sumByTag.values()) + catch (IOException e) { - long sumPeriod = (sum.getEndTimeStamp() - sum.getStartTimeStamp()); - if (verbose) - { - System.out.print(inputFile.getName()); - System.out.print(", "); - logHistogramForVerbose(System.out, sum, i++, outputValueUnitRatio); - } - sum.setEndTimeStamp(0); - sum.setStartTimeStamp(Long.MAX_VALUE); - maxPeriod = Math.max(maxPeriod, sumPeriod); + throw new RuntimeException(e); } - period += maxPeriod; + } if (ignoreTimeStamps) { diff --git a/src/main/java/UnionHistogramLogs.java b/src/main/java/UnionHistogramLogs.java index 46f5cc6..5966163 100644 --- a/src/main/java/UnionHistogramLogs.java +++ b/src/main/java/UnionHistogramLogs.java @@ -6,9 +6,7 @@ import psy.lob.saw.OrderedHistogramLogReader; import psy.lob.saw.UnionHistograms; -import java.io.File; -import java.io.FileOutputStream; -import java.io.PrintStream; +import java.io.*; import java.util.*; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -152,10 +150,13 @@ public void run() List ins = new ArrayList<>(); for (File inputFile : inputFiles) { - ins.add(new HistogramIterator( - new OrderedHistogramLogReader(inputFile, start, end), - inputFilesTags.get(inputFile), - relative)); + try(InputStream inputStream = new FileInputStream(inputFile)) + { + ins.add(new HistogramIterator( + new OrderedHistogramLogReader(inputStream, start, end), + inputFilesTags.get(inputFile), + relative)); + } } UnionHistograms unionHistograms = new UnionHistograms(verbose, System.out, ins, new HistogramSink() { diff --git a/src/main/java/psy/lob/saw/CsvConverter.java b/src/main/java/psy/lob/saw/CsvConverter.java new file mode 100644 index 0000000..d905146 --- /dev/null +++ b/src/main/java/psy/lob/saw/CsvConverter.java @@ -0,0 +1,44 @@ +package psy.lob.saw; + +import org.HdrHistogram.Histogram; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.io.PrintStream; +import java.util.Locale; + +public class CsvConverter +{ + private final InputStream inputStream; + private final PrintStream outputStream; + + public CsvConverter(InputStream inputStream, PrintStream outputStream) + { + this.inputStream = inputStream; + this.outputStream = outputStream; + } + + public void convert() throws FileNotFoundException + { + OrderedHistogramLogReader reader = new OrderedHistogramLogReader(inputStream); + outputStream.println( + "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); + while (reader.hasNext()) + { + Histogram interval = (Histogram) reader.nextIntervalHistogram(); + outputStream.printf(Locale.US, + "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", + interval.getStartTimeStamp() / 1000.0, + interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), + interval.getTotalCount(), interval.getMinValue(), + (long) interval.getMean(), + interval.getValueAtPercentile(50), + interval.getValueAtPercentile(90), + interval.getValueAtPercentile(95), + interval.getValueAtPercentile(99), + interval.getValueAtPercentile(99.9), + interval.getValueAtPercentile(99.99), + interval.getMaxValue()); + } + } +} diff --git a/src/main/java/psy/lob/saw/HistogramsSplitter.java b/src/main/java/psy/lob/saw/HistogramsSplitter.java new file mode 100644 index 0000000..a2650a3 --- /dev/null +++ b/src/main/java/psy/lob/saw/HistogramsSplitter.java @@ -0,0 +1,81 @@ +package psy.lob.saw; + +import org.HdrHistogram.Histogram; +import org.HdrHistogram.HistogramLogWriter; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Predicate; + +import static psy.lob.saw.HdrHistogramUtil.logHistogramForVerbose; + +public class HistogramsSplitter +{ + private final String inputFileName; + private final InputStream inputStream; + private final double start; + private final double end; + private final boolean verbose; + private final Predicate tagExclusionPredicate; + private final Path outputDirectory; + + public HistogramsSplitter(String inputFileName, InputStream inputStream, + double start, double end, + boolean verbose, Predicate tagExclusionPredicate, Path outputDirectory) + { + this.inputFileName = inputFileName; + this.inputStream = inputStream; + this.start = start; + this.end = end; + this.verbose = verbose; + this.tagExclusionPredicate = tagExclusionPredicate; + this.outputDirectory = outputDirectory; + } + + public Collection split() throws FileNotFoundException + { + OrderedHistogramLogReader reader = new OrderedHistogramLogReader( + inputStream, + start, + end, + tagExclusionPredicate); + Map outputFileByTag = new HashMap<>(); + Map writerByPath = new HashMap<>(); + Histogram interval; + int i = 0; + while (reader.hasNext()) + { + interval = (Histogram) reader.nextIntervalHistogram(); + if (interval == null) + { + continue; + } + String ntag = interval.getTag(); + if (tagExclusionPredicate.test(ntag)) + { + throw new IllegalStateException("Should be filtered upfront by the reader"); + } + if (this.verbose) + { + logHistogramForVerbose(System.out, interval, i++); + } + File outputFile = outputFileByTag.computeIfAbsent( + (ntag == null) ? "default" : ntag, + tag -> outputDirectory.resolve(tag + "." + inputFileName).toFile()); + HistogramLogWriter writer = writerByPath.computeIfAbsent(outputFile, o -> createWriterForTag(reader, o)); + writer.outputIntervalHistogram(interval); + } + return outputFileByTag.values(); + } + + private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, File outputFile) + { + String comment = "Splitting of:" + inputFileName + " start:" + start + " end:" + end; + return HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); + } +} \ No newline at end of file diff --git a/src/main/java/psy/lob/saw/OrderedHistogramLogReader.java b/src/main/java/psy/lob/saw/OrderedHistogramLogReader.java index f3c02fc..17df9aa 100644 --- a/src/main/java/psy/lob/saw/OrderedHistogramLogReader.java +++ b/src/main/java/psy/lob/saw/OrderedHistogramLogReader.java @@ -10,8 +10,8 @@ import org.HdrHistogram.EncodableHistogram; import java.io.Closeable; -import java.io.File; import java.io.FileNotFoundException; +import java.io.InputStream; import java.util.function.Predicate; import java.util.zip.DataFormatException; @@ -143,26 +143,26 @@ public boolean onException(Throwable t) private EncodableHistogram nextHistogram; private boolean inRange = true; - public OrderedHistogramLogReader(final File inputFile) throws FileNotFoundException + public OrderedHistogramLogReader(final InputStream inputStream) throws FileNotFoundException { - this(inputFile, 0.0, Long.MAX_VALUE * 1.0, s -> false, false); + this(inputStream, 0.0, Long.MAX_VALUE * 1.0, s -> false, false); } - public OrderedHistogramLogReader(File inputFile, double start, double end) throws FileNotFoundException + public OrderedHistogramLogReader(InputStream inputStream, double start, double end) throws FileNotFoundException { - this(inputFile, start, end, s -> false, false); + this(inputStream, start, end, s -> false, false); } - public OrderedHistogramLogReader(File inputFile, double start, double end, Predicate shouldExcludeTag) + public OrderedHistogramLogReader(InputStream inputStream, double start, double end, Predicate shouldExcludeTag) throws FileNotFoundException { - this(inputFile, start, end, shouldExcludeTag, false); + this(inputStream, start, end, shouldExcludeTag, false); } /** * Constructs a new OrderedHistogramLogReader that produces intervals read from the specified file. * - * @param inputFile The File to read from + * @param inputStream The data stream to read from * @param rangeStartTimeSec * @param rangeEndTimeSec * @param shouldExcludeTag predicate returns true is tag should be skipped @@ -170,12 +170,12 @@ public OrderedHistogramLogReader(File inputFile, double start, double end, Predi * @throws FileNotFoundException when unable to find inputFile */ public OrderedHistogramLogReader( - final File inputFile, + final InputStream inputStream, double rangeStartTimeSec, double rangeEndTimeSec, Predicate shouldExcludeTag, boolean absolute) throws FileNotFoundException { - scanner = new HistogramLogScanner(inputFile); + scanner = new HistogramLogScanner(inputStream); this.rangeStartTimeSec = rangeStartTimeSec; this.rangeEndTimeSec = rangeEndTimeSec; this.absolute = absolute;