From 05ede04006ecd49490b26098dee3e4b21e81c925 Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Fri, 6 Jun 2025 10:39:49 -0700 Subject: [PATCH 01/10] #10: add new benchmarks, run sensors, update detection script and tests --- CMakeLists.txt | 1 + detection/core/SlowNodeDetector.py | 7 +- detection/detect_slow_nodes.py | 4 + detection/utils/Parse.py | 62 ++-- src/benchmarks.cc | 289 ++++++++++++++++++ src/benchmarks.h | 46 +++ src/ops.h | 32 ++ src/ops_impl.h | 139 +++++++++ src/sensors.cc | 1 + src/slow_node.cc | 108 +------ tests/logs/slownode.log | 117 ++++--- .../unit/detection/test_slow_node_detector.py | 4 +- 12 files changed, 642 insertions(+), 168 deletions(-) create mode 100644 src/benchmarks.cc create mode 100644 src/benchmarks.h create mode 100644 src/ops.h create mode 100644 src/ops_impl.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a0a964..3ac83db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ set(SLOW_NODE_EXE slow_node) add_executable( ${SLOW_NODE_EXE} "${CMAKE_SOURCE_DIR}/src/slow_node.cc" + "${CMAKE_SOURCE_DIR}/src/benchmarks.cc" "${CMAKE_SOURCE_DIR}/src/sensors.cc" "${CMAKE_SOURCE_DIR}/src/freq.cc" ) diff --git a/detection/core/SlowNodeDetector.py b/detection/core/SlowNodeDetector.py index 43b8328..bb69fcc 100644 --- a/detection/core/SlowNodeDetector.py +++ b/detection/core/SlowNodeDetector.py @@ -35,7 +35,7 @@ class SlowNodeDetector: """ def __init__( - self, path, sensors, num_nodes, pct, spn, rpn, plot_rank_breakdowns): + self, path, sensors, num_nodes, pct, benchmark, type, spn, rpn, plot_rank_breakdowns): # Create empty dicts for storing data self.__rank_times = {} self.__rank_breakdowns = {} @@ -49,6 +49,8 @@ def __init__( self.__sensors_output_file = sensors self.__num_nodes = int(num_nodes) if num_nodes is not None else None self.__threshold_pct = float(pct) + self.__benchmark = benchmark + self.__datatype = type self.__spn = int(spn) self.__rpn = int(rpn) self.__rps = self.__rpn / self.__spn @@ -87,7 +89,7 @@ def __parseOutput(self): """Parses text output from slow_node.cc""" self.__rank_times, \ self.__rank_breakdowns, \ - self.__rank_to_node_map = parseOutput(self.__filepath) + self.__rank_to_node_map = parseOutput(self.__filepath, self.__benchmark, self.__datatype) self.__num_ranks = len(self.__rank_times) @@ -291,6 +293,7 @@ def detect(self, print_results=True): if print_results: s = self.__s(slow_rank_ids) n = len(str(abs(int(self.__num_ranks)))) + print(f"\nPrinting analysis from {self.__benchmark}_{self.__datatype} benchmark...") print("\n----------------------------------------------------------") print("Across-Rank Analysis") print() diff --git a/detection/detect_slow_nodes.py b/detection/detect_slow_nodes.py index cdc4bef..510451a 100644 --- a/detection/detect_slow_nodes.py +++ b/detection/detect_slow_nodes.py @@ -17,6 +17,8 @@ def main(): parser.add_argument('-s', '--sensors', help='Absolute or relative path to the sensors file that will be analyzed', default=None) parser.add_argument('-N', '--num_nodes', help='The number of nodes required by the application', default=None) parser.add_argument('-t', '--threshold', help='Percentage above average time that indicates a "slow" rank', default=0.05) + parser.add_argument('-b', '--benchmark', help='Benchmark to analyze: [level1, level2, level3, dpotrf]', default='level3') + parser.add_argument('-d', '--datatype', help='Datatype of benchmark to analyze: [double, complex]', default='double') parser.add_argument('-spn', '--spn', help='Number of sockets per node', default=2) parser.add_argument('-rpn', '--rpn', help='Number of ranks per node', default=48) parser.add_argument('-p', '--plot_all_ranks', action='store_true', help='Plot the breakdowns for every rank') @@ -30,6 +32,8 @@ def main(): sensors=sensors_filepath, num_nodes=args.num_nodes, pct=args.threshold, + benchmark=args.benchmark, + type=args.datatype, spn=args.spn, rpn=args.rpn, plot_rank_breakdowns=args.plot_all_ranks) diff --git a/detection/utils/Parse.py b/detection/utils/Parse.py index 4054a93..3101033 100644 --- a/detection/utils/Parse.py +++ b/detection/utils/Parse.py @@ -7,38 +7,46 @@ def matchRegex(pattern: str, line: str): return tuple(match.groups()) raise RuntimeError(f"regex matching failed on line {line}") -def parseOutput(slownode_file): +def parseOutput(slownode_file, benchmark, datatype): """Parses text output from slow_node.cc""" rank_times = {} rank_breakdowns = {} rank_to_node_map = {} + is_parsing=False with open(slownode_file, "r") as output: for line in output: - if line.startswith("gather"): - # splits: ['gather', rank_info, total_time, 'breakdown', [times]] - splits = line.split(":") - - # 1. Determine the Rank ID (and node name, if present) - raw_rank_info = splits[1].strip() - # raw_rank_info = 'rank_id (node)' - rank_info = re.findall( - r"(\d+)\s+\(([^)]+)\)", - raw_rank_info - )[0] - rank_id = int(rank_info[0]) - node_name = rank_info[1] - rank_to_node_map[rank_id] = node_name - - # 2. Get the total time for the current rank - total_time = float(splits[2].strip()) - - # 3. Isolate the times for each iteration on the current rank - breakdown = splits[4].strip() - breakdown_list = [float(t) for t in breakdown.split(" ")] - - # Populate rank data dicts - rank_times[rank_id] = total_time - rank_breakdowns[rank_id] = breakdown_list + if line.startswith(f"{benchmark}_{datatype}"): + is_parsing = True + + if is_parsing: + if line.startswith("gather"): + # splits: ['gather', rank_info, total_time, 'breakdown', [times]] + splits = line.split(":") + + # 1. Determine the Rank ID (and node name, if present) + raw_rank_info = splits[1].strip() + # raw_rank_info = 'rank_id (node)' + rank_info = re.findall( + r"(\d+)\s+\(([^)]+)\)", + raw_rank_info + )[0] + rank_id = int(rank_info[0]) + node_name = rank_info[1] + rank_to_node_map[rank_id] = node_name + + # 2. Get the total time for the current rank + total_time = float(splits[2].strip()) + + # 3. Isolate the times for each iteration on the current rank + breakdown = splits[4].strip() + breakdown_list = [float(t) for t in breakdown.split(" ")] + + # Populate rank data dicts + rank_times[rank_id] = total_time + rank_breakdowns[rank_id] = breakdown_list + + elif line.strip() == "": + is_parsing = False return rank_times, rank_breakdowns, rank_to_node_map @@ -52,7 +60,7 @@ def parseSensors(sensors_file): with open(sensors_file, 'r') as sensor_data: for line in sensor_data: if line.startswith("Node"): - pattern = r"Node (\w+), Socket (\d+), Core (\d+): (\d+)(?:°C| C), (\d+) KHz" + pattern = r"Node (\w+), Socket (\d+), Core (\d+): (\d+)(?:°C| C), (?:-|)(\d+) KHz" node_name, \ socket_str, \ diff --git a/src/benchmarks.cc b/src/benchmarks.cc new file mode 100644 index 0000000..834726c --- /dev/null +++ b/src/benchmarks.cc @@ -0,0 +1,289 @@ +#include +#include +#include + +#include "ops.h" +#include "benchmarks.h" + +#include + +namespace benchmarks { + +template <> +std::string typeToString() { + return "double"; +} + +template <> +std::string typeToString>() { + return "complex"; +} + +std::string benchmarkToString(const benchmark_types& b) { + switch (b) { + case level1: return "level1"; + case level2: return "level2"; + case level3: return "level3"; + case dpotrf: return "dpotrf"; + default: return "unknown"; + } +} + +template +benchmark_results_t runBenchmarkLevel1(int N, int iters) { + Kokkos::View x("x", N); + Kokkos::View y("y", N); + + T* x_ptr = x.data(); + T* y_ptr = y.data(); + + Kokkos::Random_XorShift64_Pool pool(123); + Kokkos::fill_random(x, pool, 10.0); + Kokkos::fill_random(y, pool, 10.0); + + std::vector iter_timings; + double total_time = 0.0; + + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < iters; i++) { + Kokkos::Timer timer; + ops::level1(N, x_ptr, y_ptr); + Kokkos::fence(); + + // Skip the first iteration + if (i > 0) { + double time = timer.seconds(); + total_time += time; + iter_timings.push_back(time); + } + } + + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + std::cout << "[level1 " << typeToString() << "] rank: " << rank << ", total_time=" << total_time << std::endl; + + return std::make_tuple(iter_timings, total_time); +} + +template +benchmark_results_t runBenchmarkLevel2(int M, int N, int iters) { + Kokkos::View x("x", M); + Kokkos::View y("y", N); + Kokkos::View A("A", M, N); + + T* x_ptr = x.data(); + T* y_ptr = y.data(); + T* A_ptr = A.data(); + + Kokkos::Random_XorShift64_Pool pool(123); + Kokkos::fill_random(x, pool, 10.0); + Kokkos::fill_random(y, pool, 10.0); + // Kokkos::fill_random(A, pool, 10.0); + + std::vector iter_timings; + double total_time = 0.0; + + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < iters; i++) { + Kokkos::Timer timer; + ops::level2(M, N, T(1.0), x_ptr, y_ptr, A_ptr, N); + Kokkos::fence(); + + // Skip the first iteration + if (i > 0) { + double time = timer.seconds(); + total_time += time; + iter_timings.push_back(time); + } + } + + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + std::cout << "[level2 " << typeToString() << "] rank: " << rank << ", total_time=" << total_time << std::endl; + + return std::make_tuple(iter_timings, total_time); +} + +template +benchmark_results_t runBenchmarkLevel3(int M, int N, int K, int iters) { + Kokkos::View A("A", M, N); + Kokkos::View B("B", N, K); + Kokkos::View C("C", M, K); + + T* A_ptr = A.data(); + T* B_ptr = B.data(); + T* C_ptr = C.data(); + + Kokkos::Random_XorShift64_Pool pool(123); + Kokkos::fill_random(A, pool, 10.0); + Kokkos::fill_random(B, pool, 10.0); + + std::vector iter_timings; + + double total_time = 0.0; + + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < iters; i++) { + Kokkos::Timer timer; + ops::level3(M, K, N, T(1.0), A_ptr, N, B_ptr, K, T(0.0), C_ptr, K); + Kokkos::fence(); + + // Skip the first iteration + if (i > 0) { + double time = timer.seconds(); + total_time += time; + iter_timings.push_back(time); + } + } + + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + std::cout << "[level3 " << typeToString() << "] rank: " << rank << ", total_time=" << total_time << std::endl; + + return std::make_tuple(iter_timings, total_time); +} + +template +benchmark_results_t runBenchmarkDPOTRF(int N, int iters) { + Kokkos::View A("A", N, N); + + Kokkos::Random_XorShift64_Pool pool(123); + Kokkos::fill_random(A, pool, 10.0); + + // Make A symmetric positive definite + Kokkos::parallel_for("MakeSPD", N, KOKKOS_LAMBDA(int i) { + for (int j = 0; j < N; j++) { + A(i, j) = A(i, j) + A(j, i); + } + }); + + Kokkos::fence(); + + T* A_ptr = A.data(); + + std::vector iter_timings; + double total_time = 0.0; + + long long N_ll = static_cast(N); + char uplo = 'L'; + + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < iters; i++) { + long long info; + Kokkos::Timer timer; + ops::dpotrf(uplo, N_ll, A_ptr, N_ll, &info); + + Kokkos::fence(); + + // Skip the first iteration + if (i > 0) { + double time = timer.seconds(); + total_time += time; + iter_timings.push_back(time); + } + } + + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + std::cout << "[dpotrf " << typeToString() << "] rank: " << rank << ", total_time=" << total_time << std::endl; + + return std::make_tuple(iter_timings, total_time); +} + +template +benchmark_results_t runBenchmark(benchmark_types b, int M, int N, int K, int iters) { + switch (b) { + case level1: + return runBenchmarkLevel1(N, iters); + case level2: + return runBenchmarkLevel2(M, N, iters); + case level3: + return runBenchmarkLevel3(M, N, K, iters); + case dpotrf: + return runBenchmarkDPOTRF(N, iters); + default: + throw std::invalid_argument("Unsupported benchmark type"); + } +} + +all_results_t runAllBenchmarks(int M, int N, int K, int iters) { + all_results_t all_results; + for (int i=0; i < benchmark_types::num_benchmarks; i++) { + auto b = static_cast(i); + std::string benchmark_str = benchmarkToString(b); + all_results[benchmark_str + "_double"] = runBenchmark(b, M, N, K, iters); + all_results[benchmark_str + "_complex"] = runBenchmark>(b, M, N, K, iters); + } + return all_results; +} + +void printBenchmarkOutput(all_results_t benchmark_results, int iters) +{ + int rank = -1; + int num_ranks = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &num_ranks); + + if (rank == 0) { + std::cout << "num_ranks: " << num_ranks << std::endl; + } + + for (const auto& [benchmark_str, benchmark_results]: benchmark_results) { + char processor_name[MPI_MAX_PROCESSOR_NAME]; + int name_len; + MPI_Get_processor_name(processor_name, &name_len); + + auto const& [iter_timings, total_time] = benchmark_results; + + std::vector all_times; + all_times.resize(num_ranks); + + std::vector all_iter_times; + all_iter_times.resize(num_ranks * iters); + + std::vector all_processor_names; + all_processor_names.resize(num_ranks * MPI_MAX_PROCESSOR_NAME); + + MPI_Gather( + &total_time, 1, MPI_DOUBLE, + &all_times[0], 1, MPI_DOUBLE, 0, + MPI_COMM_WORLD + ); + + MPI_Gather( + &iter_timings[0], iters, MPI_DOUBLE, + &all_iter_times[0], iters, MPI_DOUBLE, 0, + MPI_COMM_WORLD + ); + + MPI_Gather( + &processor_name, MPI_MAX_PROCESSOR_NAME, MPI_CHAR, + &all_processor_names[0], MPI_MAX_PROCESSOR_NAME, MPI_CHAR, 0, + MPI_COMM_WORLD + ); + + if (rank == 0) { + int cur_rank = 0; + int cur = 0; + std::cout << std::endl << benchmark_str << std::endl; + for (auto&& time : all_times) { + std::cout << "gather: " << cur_rank << " (" + << std::string(&all_processor_names[cur_rank * MPI_MAX_PROCESSOR_NAME]) + << "): " << time << ": breakdown: "; + for (int i = cur; i < iters + cur; i++) { + std::cout << all_iter_times[i] << " "; + } + std::cout << std::endl; + cur += iters; + cur_rank++; + } + } + } +} +} // end namespace benchmarks diff --git a/src/benchmarks.h b/src/benchmarks.h new file mode 100644 index 0000000..8591df8 --- /dev/null +++ b/src/benchmarks.h @@ -0,0 +1,46 @@ +#ifndef SRC_BENCHMARKS_H +#define SRC_BENCHMARKS_H + +#include +#include + +namespace benchmarks { + +using benchmark_results_t = std::tuple, double>; +using all_results_t = std::unordered_map; + +enum benchmark_types { + level1, + level2, + level3, + dpotrf, + num_benchmarks +}; + +template +std::string typeToString(); + +std::string benchmarkToString(const benchmark_types& b); + +template +benchmark_results_t runBenchmarkLevel1(int N, int iters); + +template +benchmark_results_t runBenchmarkLevel2(int M, int N, int iters); + +template +benchmark_results_t runBenchmarkLevel3(int M, int N, int K, int iters); + +template +benchmark_results_t runBenchmarkDPOTRF(int N, int iters); + +template +benchmark_results_t runBenchmark(benchmark_types b, int M, int N, int K, int iters); + +all_results_t runAllBenchmarks(int M, int N, int K, int iters); + +void printBenchmarkOutput(all_results_t benchmark_results, int iters); + +} // end namespace benchmarks + +#endif // SRC_BENCHMARKS_H diff --git a/src/ops.h b/src/ops.h new file mode 100644 index 0000000..4327f0b --- /dev/null +++ b/src/ops.h @@ -0,0 +1,32 @@ +#ifndef SRC_OPS_H +#define SRC_OPS_H + +namespace ops { + +template +constexpr bool isDouble(); + +template +constexpr bool isComplex(); + +template +void level1(int N, const T* x, const T* y); + +template +void level2(int M, int N, T alpha, const T* x, const T* y, T* A, int lda); + +template +void level3( + int M, int N, int K, + T alpha, const T* A, int lda, + const T* B, int ldb, + T beta, T* C, int ldc); + +template +void dpotrf(char uplo, long long n, T* a, long long lda, long long* info); + +} // end namespace ops + +#include "ops_impl.h" + +#endif // SRC_OPS_H diff --git a/src/ops_impl.h b/src/ops_impl.h new file mode 100644 index 0000000..5a051df --- /dev/null +++ b/src/ops_impl.h @@ -0,0 +1,139 @@ +#ifndef SRC_OPS_IMPL_H +#define SRC_OPS_IMPL_H + +#include +#include + +#include + +#include "ops.h" + +namespace ops { + +/////////////////////////////////////////////////////////////////////////////// +// Helpers + +template +constexpr bool isDouble() { + if constexpr (std::is_same_v) { + return true; + } + return false; +} + +template +constexpr bool isComplex() { + if constexpr (std::is_same_v>) { + return true; + } + return false; +} + +static void warnWrongType(const std::string& benchmark) { + std::cout << "Warning: Type must be either double or Kokkos::complex. Skipping " << benchmark << " benchmark." << std::endl; +} + +/////////////////////////////////////////////////////////////////////////////// +// LEVEL 1 + +template +void level1(int N, const T* x, const T* y) { + if constexpr (isDouble()) { + cblas_ddot(N, x, 1, y, 1); + } else if constexpr (isComplex) { + T result; + cblas_zdotu_sub(N, x, 1, y, 1, &result); + } else { + warnWrongType("level1"); + } +} + +/////////////////////////////////////////////////////////////////////////////// +// LEVEL 2 + +template +void level2( + int M, + int N, + T alpha, + const T* x, + const T* y, + T* A, + int lda) +{ + if constexpr (isDouble()) { + cblas_dger(CblasRowMajor, M, N, alpha, x, 1, y, 1, A, lda); + } else if constexpr (isComplex()) { + cblas_zgeru( + CblasRowMajor, + M, N, + reinterpret_cast(&alpha), + reinterpret_cast(x), 1, + reinterpret_cast(y), 1, + reinterpret_cast(A), lda + ); + } else { + warnWrongType("level2"); + } +} + +/////////////////////////////////////////////////////////////////////////////// +// LEVEL 3 + +template +void level3( + int M, + int N, + int K, + T alpha, + const T* A, + int lda, + const T* B, + int ldb, + T beta, + T* C, + int ldc) +{ + if constexpr (isDouble()) { + cblas_dgemm( + CblasRowMajor, CblasNoTrans, CblasNoTrans, + M, N, K, + alpha, + A, lda, + B, ldb, + beta, + C, ldc + ); + } else if constexpr (isComplex()) { + cblas_zgemm( + CblasRowMajor, CblasNoTrans, CblasNoTrans, + M, N, K, + reinterpret_cast(&alpha), + reinterpret_cast(A), lda, + reinterpret_cast(B), ldb, + reinterpret_cast(&beta), + reinterpret_cast(C), ldc); + } else { + warnWrongType("level3"); + } + + +} + +/////////////////////////////////////////////////////////////////////////////// +// DPOTRF + +template +void dpotrf(char uplo, long long n, T* a, long long lda, long long* info) { + if constexpr (isDouble()) { + dpotrf_(&uplo, &n, a, &lda, info); + } else if constexpr (isComplex()) { + zpotrf_(&uplo, &n, reinterpret_cast(a), &lda, info); + } else { + warnWrongType("dpotrf"); + } +} + +} // end namespace ops + +#endif // SRC_OPS_IMPL_H diff --git a/src/sensors.cc b/src/sensors.cc index a45501c..a3d9d11 100644 --- a/src/sensors.cc +++ b/src/sensors.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/slow_node.cc b/src/slow_node.cc index f312aa2..8206aea 100644 --- a/src/slow_node.cc +++ b/src/slow_node.cc @@ -1,7 +1,7 @@ #include "sensors.h" +#include "benchmarks.h" -#include #include #include @@ -11,56 +11,6 @@ static int M = 128; static int N = 128; static int K = 128; -std::tuple, double> runBenchmark() { - Kokkos::View A("A", M, N); - Kokkos::View B("B", N, K); - Kokkos::View C("C", M, K); - - double* A_ptr = A.data(); - double* B_ptr = B.data(); - double* C_ptr = C.data(); - - Kokkos::Random_XorShift64_Pool pool(123); - Kokkos::fill_random(A, pool, 10.0); - Kokkos::fill_random(B, pool, 10.0); - - std::vector iter_timings; - - double total_time = 0.0; - - MPI_Barrier(MPI_COMM_WORLD); - - for (int i = 0; i < iters; i++) { - Kokkos::Timer timer; - cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - M, // number of rows in C (and A) - K, // number of columns in C (and B) - N, // shared inner dimension (columns of A, rows of B) - 1.0, // alpha - A_ptr, // matrix A pointer - N, // leading dimension of A (because A is M×N) - B_ptr, // matrix B pointer - K, // leading dimension of B (because B is N×K) - 0.0, // beta - C_ptr, // matrix C pointer - K); // leading dimension of C (because C is M×K) - Kokkos::fence(); - // Do not count the first iteration - if (i > 0) { - double time = timer.seconds(); - total_time += time; - iter_timings.push_back(time); - } - } - - int rank = -1; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - std::cout << "rank: " << rank << ", total_time=" << total_time << std::endl; - - return std::make_tuple(iter_timings, total_time); -} - int main(int argc, char** argv) { if (argc > 1) { iters = atoi(argv[1]) + 1; // add one iteration since we will drop the first one @@ -71,65 +21,15 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); Kokkos::initialize(argc, argv); - int rank = -1; - int num_ranks = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &num_ranks); - char processor_name[MPI_MAX_PROCESSOR_NAME]; int name_len; MPI_Get_processor_name(processor_name, &name_len); + // Loop through all available benchmarks sensors::runSensorsAndReduceOutput(processor_name, "pre"); - auto const& [iter_timings, total_time] = runBenchmark(); + auto output = benchmarks::runAllBenchmarks(M, N, K, iters); sensors::runSensorsAndReduceOutput(processor_name, "post"); - - std::vector all_times; - all_times.resize(num_ranks); - - std::vector all_iter_times; - all_iter_times.resize(num_ranks * iters); - - std::vector all_processor_names; - all_processor_names.resize(num_ranks * MPI_MAX_PROCESSOR_NAME); - - if (rank == 0) { - std::cout << "num_ranks: " << num_ranks << std::endl; - } - - MPI_Gather( - &total_time, 1, MPI_DOUBLE, - &all_times[0], 1, MPI_DOUBLE, 0, - MPI_COMM_WORLD - ); - - MPI_Gather( - &iter_timings[0], iters, MPI_DOUBLE, - &all_iter_times[0], iters, MPI_DOUBLE, 0, - MPI_COMM_WORLD - ); - - MPI_Gather( - &processor_name, MPI_MAX_PROCESSOR_NAME, MPI_CHAR, - &all_processor_names[0], MPI_MAX_PROCESSOR_NAME, MPI_CHAR, 0, - MPI_COMM_WORLD - ); - - if (rank == 0) { - int cur_rank = 0; - int cur = 0; - for (auto&& time : all_times) { - std::cout << "gather: " << cur_rank << " (" - << std::string(&all_processor_names[cur_rank * MPI_MAX_PROCESSOR_NAME]) - << "): " << time << ": breakdown: "; - for (int i = cur; i < iters + cur; i++) { - std::cout << all_iter_times[i] << " "; - } - std::cout << std::endl; - cur += iters; - cur_rank++; - } - } + benchmarks::printBenchmarkOutput(output, iters); Kokkos::finalize(); MPI_Finalize(); diff --git a/tests/logs/slownode.log b/tests/logs/slownode.log index 66678b3..a11dcd4 100644 --- a/tests/logs/slownode.log +++ b/tests/logs/slownode.log @@ -2,37 +2,86 @@ iters: 100, M=N=K=128 iters: 100, M=N=K=128 iters: 100, M=N=K=128 iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -rank: 0, total_time=1.00737 -rank: 3, total_time=1.1036 -rank: 8, total_time=1.20108 -rank: 5, total_time=1.20714 -rank: 9, total_time=6.61778 -rank: 7, total_time=6.6205 -rank: 10, total_time=6.67709 -rank: 11, total_time=6.73952 -rank: 4, total_time=6.74128 -rank: 2, total_time=6.84974 -rank: 6, total_time=6.89012 -rank: 1, total_time=6.95131 -Wrote sensor data for ozark to "sensors_output/sensors_ozark.log" -num_ranks: 12 -gather: 0 (ozark): 1.00737: breakdown: 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 -gather: 1 (ozark): 6.95131: breakdown: 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 -gather: 2 (ozark): 6.84974: breakdown: 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 -gather: 3 (ozark): 1.1036: breakdown: 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 -gather: 4 (ozark): 6.74128: breakdown: 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 -gather: 5 (ozark): 1.20714: breakdown: 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 -gather: 6 (ozark): 6.89012: breakdown: 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 -gather: 7 (ozark): 6.6205: breakdown: 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 -gather: 8 (ozark): 1.20108: breakdown: 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 -gather: 9 (ozark): 6.61778: breakdown: 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 -gather: 10 (ozark): 6.67709: breakdown: 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 -gather: 11 (ozark): 6.73952: breakdown: 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 +Wrote sensor data to "sensors_pre.log" +[level1 double] rank: 2, total_time=2.35e-05 +[level1 double] rank: 1, total_time=2.3465e-05 +[level1 double] rank: 3, total_time=2.3176e-05 +[level1 double] rank: 0, total_time=2.6729e-05 +[level1 complex] rank: 0, total_time=1.8592e-05 +[level1 complex] rank: 3, total_time=1.8519e-05 +[level1 complex] rank: 2, total_time=1.9089e-05 +[level1 complex] rank: 1, total_time=2.1363e-05 +[level2 double] rank: 3, total_time=0.000377028 +[level2 double] rank: 2, total_time=0.000375657 +[level2 double] rank: 0, total_time=0.000378346 +[level2 double] rank: 1, total_time=0.000377205 +[level2 complex] rank: 1, total_time=0.000860855 +[level2 complex] rank: 3, total_time=0.000864736 +[level2 complex] rank: 2, total_time=0.000864709 +[level2 complex] rank: 0, total_time=0.000867368 +[level3 double] rank: 0, total_time=0.013612 +[level3 double] rank: 1, total_time=0.0136195 +[level3 double] rank: 2, total_time=0.0136247 +[level3 double] rank: 3, total_time=0.0136833 +[level3 complex] rank: 3, total_time=0.0535708 +[level3 complex] rank: 1, total_time=0.0536701 +[level3 complex] rank: 2, total_time=0.0538132 +[level3 complex] rank: 0, total_time=0.0538252 +[dpotrf double] rank: 2, total_time=3.607e-05 +[dpotrf double] rank: 3, total_time=3.5959e-05 +[dpotrf double] rank: 0, total_time=3.568e-05 +[dpotrf double] rank: 1, total_time=3.6751e-05 +[dpotrf complex] rank: 2, total_time=0.0010499 +[dpotrf complex] rank: 1, total_time=0.00105277 +[dpotrf complex] rank: 3, total_time=0.00105296 +[dpotrf complex] rank: 0, total_time=0.00106083 +Wrote sensor data to "sensors_post.log" +num_ranks: 4 + +dpotrf_complex +gather: 0 (ozark): 0.00106083: breakdown: 1.2762e-05 1.1106e-05 1.0822e-05 1.0656e-05 1.0811e-05 1.0742e-05 1.0703e-05 1.069e-05 1.0638e-05 1.0725e-05 1.0536e-05 1.7613e-05 1.2004e-05 1.0648e-05 1.0615e-05 1.0659e-05 1.07e-05 1.0593e-05 1.0581e-05 1.0604e-05 1.0641e-05 1.0629e-05 1.0607e-05 1.0568e-05 1.0478e-05 1.0637e-05 1.0607e-05 1.0511e-05 1.0555e-05 1.0709e-05 1.0533e-05 1.0573e-05 1.0611e-05 1.06e-05 1.0541e-05 1.0567e-05 1.0562e-05 1.0609e-05 1.0652e-05 1.0511e-05 1.0596e-05 1.0561e-05 1.0536e-05 1.0604e-05 1.0589e-05 1.0563e-05 1.0617e-05 1.06e-05 1.055e-05 1.063e-05 1.0494e-05 1.0502e-05 1.0559e-05 1.062e-05 1.0596e-05 1.052e-05 1.0565e-05 1.0513e-05 1.0587e-05 1.0604e-05 1.0517e-05 1.0588e-05 1.0612e-05 1.0583e-05 1.0628e-05 1.0528e-05 1.0574e-05 1.0555e-05 1.0527e-05 1.0609e-05 1.0624e-05 1.0526e-05 1.061e-05 1.0587e-05 1.0579e-05 1.0555e-05 1.0519e-05 1.0593e-05 1.0607e-05 1.0634e-05 1.0571e-05 1.0636e-05 1.0644e-05 1.0575e-05 1.0572e-05 1.059e-05 1.0548e-05 1.063e-05 1.0611e-05 1.0667e-05 1.061e-05 1.0701e-05 1.0654e-05 1.0705e-05 1.0666e-05 1.0587e-05 1.0671e-05 1.0655e-05 1.0671e-05 1.0326e-321 +gather: 1 (ozark): 0.00105277: breakdown: 1.1991e-05 1.089e-05 1.079e-05 1.0664e-05 1.061e-05 1.0528e-05 1.0521e-05 1.0519e-05 1.0559e-05 1.0628e-05 1.0618e-05 1.5703e-05 1.0782e-05 1.0575e-05 1.0636e-05 1.0599e-05 1.0622e-05 1.0719e-05 1.0554e-05 1.0471e-05 1.0515e-05 1.0628e-05 1.0491e-05 1.0667e-05 1.0623e-05 1.0477e-05 1.0521e-05 1.0472e-05 1.0525e-05 1.0514e-05 1.0532e-05 1.0529e-05 1.0596e-05 1.063e-05 1.0552e-05 1.0611e-05 1.0528e-05 1.0601e-05 1.0565e-05 1.0533e-05 1.0489e-05 1.0516e-05 1.055e-05 1.0492e-05 1.0569e-05 1.0574e-05 1.0488e-05 1.0563e-05 1.0611e-05 1.0617e-05 1.0565e-05 1.0599e-05 1.0536e-05 1.0573e-05 1.0533e-05 1.0526e-05 1.067e-05 1.0496e-05 1.0475e-05 1.0499e-05 1.0539e-05 1.0544e-05 1.0575e-05 1.0539e-05 1.058e-05 1.0572e-05 1.0632e-05 1.0662e-05 1.0576e-05 1.0603e-05 1.0485e-05 1.0559e-05 1.0625e-05 1.0561e-05 1.0521e-05 1.0556e-05 1.0563e-05 1.0552e-05 1.0518e-05 1.0484e-05 1.051e-05 1.0564e-05 1.0645e-05 1.0522e-05 1.0587e-05 1.055e-05 1.0528e-05 1.0578e-05 1.0553e-05 1.0541e-05 1.0489e-05 1.054e-05 1.0692e-05 1.0586e-05 1.0536e-05 1.0527e-05 1.0521e-05 1.0466e-05 1.056e-05 1.0326e-321 +gather: 2 (ozark): 0.0010499: breakdown: 1.2737e-05 1.1093e-05 1.084e-05 1.071e-05 1.0682e-05 1.0697e-05 1.068e-05 1.0594e-05 1.0543e-05 1.0564e-05 1.0641e-05 1.3895e-05 1.0677e-05 1.0648e-05 1.0611e-05 1.053e-05 1.0576e-05 1.0542e-05 1.0563e-05 1.0571e-05 1.0584e-05 1.0496e-05 1.0561e-05 1.0516e-05 1.0504e-05 1.0598e-05 1.049e-05 1.0543e-05 1.0564e-05 1.055e-05 1.0487e-05 1.0514e-05 1.0499e-05 1.0573e-05 1.0565e-05 1.054e-05 1.0576e-05 1.0574e-05 1.0548e-05 1.0575e-05 1.0544e-05 1.0572e-05 1.0517e-05 1.0454e-05 1.0595e-05 1.0503e-05 1.0415e-05 1.0499e-05 1.0466e-05 1.049e-05 1.0472e-05 1.0524e-05 1.0504e-05 1.0553e-05 1.0536e-05 1.0523e-05 1.05e-05 1.0514e-05 1.0493e-05 1.0603e-05 1.061e-05 1.0506e-05 1.0595e-05 1.0495e-05 1.0472e-05 1.0542e-05 1.0515e-05 1.0502e-05 1.055e-05 1.0528e-05 1.0521e-05 1.0463e-05 1.0536e-05 1.0514e-05 1.0562e-05 1.0451e-05 1.0563e-05 1.0509e-05 1.0514e-05 1.0589e-05 1.0515e-05 1.0495e-05 1.0509e-05 1.0523e-05 1.0461e-05 1.0514e-05 1.0481e-05 1.0469e-05 1.0566e-05 1.0594e-05 1.0481e-05 1.0552e-05 1.055e-05 1.0549e-05 1.0544e-05 1.0493e-05 1.0461e-05 1.0583e-05 1.0495e-05 1.0326e-321 +gather: 3 (ozark): 0.00105296: breakdown: 1.2346e-05 1.1251e-05 1.0981e-05 1.0918e-05 1.0768e-05 1.0651e-05 1.0801e-05 1.0648e-05 1.0674e-05 1.0583e-05 1.054e-05 1.4257e-05 1.0916e-05 1.0637e-05 1.0546e-05 1.0602e-05 1.0595e-05 1.0568e-05 1.0677e-05 1.0658e-05 1.0597e-05 1.0599e-05 1.0588e-05 1.0549e-05 1.0599e-05 1.0492e-05 1.0531e-05 1.0589e-05 1.0547e-05 1.0585e-05 1.0636e-05 1.049e-05 1.0568e-05 1.0574e-05 1.0561e-05 1.0567e-05 1.0551e-05 1.0584e-05 1.0553e-05 1.0523e-05 1.053e-05 1.0584e-05 1.0539e-05 1.0521e-05 1.0567e-05 1.0472e-05 1.0534e-05 1.0611e-05 1.0491e-05 1.0504e-05 1.0609e-05 1.0556e-05 1.0494e-05 1.0544e-05 1.0523e-05 1.0557e-05 1.0548e-05 1.0546e-05 1.0564e-05 1.0585e-05 1.0552e-05 1.0591e-05 1.0507e-05 1.0513e-05 1.0521e-05 1.0524e-05 1.0575e-05 1.054e-05 1.063e-05 1.0557e-05 1.056e-05 1.0516e-05 1.0524e-05 1.0612e-05 1.0487e-05 1.0491e-05 1.0566e-05 1.0514e-05 1.0542e-05 1.0687e-05 1.0513e-05 1.0495e-05 1.054e-05 1.0484e-05 1.0528e-05 1.0496e-05 1.0573e-05 1.0505e-05 1.0566e-05 1.0461e-05 1.0563e-05 1.0576e-05 1.0481e-05 1.0578e-05 1.0549e-05 1.052e-05 1.0666e-05 1.0529e-05 1.0545e-05 1.0326e-321 + +dpotrf_double +gather: 0 (ozark): 3.568e-05: breakdown: 2.039e-06 7.06e-07 4.16e-07 3.82e-07 3.66e-07 3.88e-07 3.83e-07 3.62e-07 3.52e-07 3.43e-07 3.42e-07 3.4e-07 3.37e-07 3.33e-07 3.47e-07 3.35e-07 3.4e-07 3.54e-07 3.43e-07 3.46e-07 3.47e-07 3.41e-07 3.38e-07 3.36e-07 3.36e-07 3.37e-07 3.34e-07 3.39e-07 3.37e-07 3.43e-07 3.4e-07 3.44e-07 3.41e-07 3.37e-07 3.35e-07 3.39e-07 3.4e-07 3.34e-07 3.39e-07 3.4e-07 3.32e-07 3.34e-07 3.37e-07 3.39e-07 3.41e-07 3.46e-07 3.35e-07 3.37e-07 3.31e-07 3.36e-07 3.35e-07 3.4e-07 3.87e-07 3.41e-07 3.32e-07 3.34e-07 3.27e-07 3.32e-07 3.22e-07 3.33e-07 3.27e-07 3.33e-07 3.32e-07 3.34e-07 3.38e-07 3.42e-07 3.5e-07 3.37e-07 3.34e-07 3.3e-07 3.31e-07 3.36e-07 3.35e-07 3.38e-07 3.32e-07 3.34e-07 3.31e-07 3.32e-07 3.34e-07 3.31e-07 3.36e-07 3.31e-07 3.33e-07 3.29e-07 3.3e-07 3.34e-07 3.36e-07 3.36e-07 3.36e-07 3.28e-07 3.34e-07 3.39e-07 3.29e-07 3.34e-07 3.24e-07 3.27e-07 3.27e-07 3.31e-07 3.33e-07 1.9812e-321 +gather: 1 (ozark): 3.6751e-05: breakdown: 1.742e-06 4.88e-07 4.09e-07 3.96e-07 3.98e-07 3.91e-07 4.11e-07 3.68e-07 3.67e-07 3.54e-07 4.01e-07 3.71e-07 3.59e-07 3.65e-07 3.51e-07 3.59e-07 3.56e-07 3.53e-07 3.63e-07 3.46e-07 3.47e-07 3.44e-07 3.47e-07 3.58e-07 3.49e-07 3.4e-07 3.45e-07 3.48e-07 3.45e-07 3.5e-07 3.5e-07 3.46e-07 3.52e-07 3.81e-07 3.59e-07 3.53e-07 3.53e-07 3.72e-07 3.43e-07 3.56e-07 3.4e-07 3.81e-07 3.54e-07 3.55e-07 3.49e-07 3.5e-07 3.49e-07 3.54e-07 3.51e-07 3.5e-07 3.49e-07 3.55e-07 3.55e-07 3.49e-07 3.49e-07 3.55e-07 3.47e-07 3.57e-07 3.56e-07 3.6e-07 3.44e-07 3.55e-07 3.53e-07 3.5e-07 3.49e-07 3.81e-07 3.63e-07 3.67e-07 3.57e-07 3.57e-07 3.42e-07 3.44e-07 3.54e-07 3.46e-07 3.6e-07 3.45e-07 3.85e-07 3.48e-07 3.57e-07 3.54e-07 3.58e-07 3.57e-07 3.45e-07 3.45e-07 3.47e-07 3.44e-07 3.45e-07 3.41e-07 3.43e-07 3.39e-07 3.48e-07 3.5e-07 3.44e-07 3.45e-07 3.42e-07 3.47e-07 3.56e-07 3.49e-07 3.44e-07 2.77171e-321 +gather: 2 (ozark): 3.607e-05: breakdown: 2.04e-06 6.22e-07 4.35e-07 3.89e-07 3.73e-07 3.76e-07 3.6e-07 3.49e-07 3.45e-07 3.47e-07 3.51e-07 3.47e-07 3.36e-07 3.39e-07 3.4e-07 3.39e-07 3.4e-07 3.4e-07 3.45e-07 3.38e-07 3.43e-07 3.48e-07 3.43e-07 3.38e-07 3.41e-07 3.44e-07 3.38e-07 3.47e-07 3.44e-07 3.4e-07 3.35e-07 3.43e-07 3.44e-07 4.92e-07 3.46e-07 3.49e-07 3.45e-07 3.48e-07 3.31e-07 3.45e-07 3.4e-07 3.47e-07 3.36e-07 3.39e-07 3.4e-07 3.35e-07 3.39e-07 3.38e-07 3.38e-07 3.38e-07 3.38e-07 3.38e-07 3.37e-07 3.35e-07 3.73e-07 3.42e-07 3.36e-07 3.4e-07 3.41e-07 3.4e-07 3.31e-07 3.34e-07 3.4e-07 3.36e-07 3.37e-07 3.49e-07 3.46e-07 3.39e-07 3.38e-07 3.32e-07 3.37e-07 3.45e-07 3.41e-07 3.39e-07 3.42e-07 3.3e-07 3.37e-07 3.4e-07 3.38e-07 3.4e-07 3.42e-07 3.41e-07 3.38e-07 3.34e-07 3.4e-07 3.41e-07 3.34e-07 3.39e-07 3.42e-07 3.36e-07 3.34e-07 3.38e-07 3.35e-07 3.36e-07 3.37e-07 3.37e-07 3.44e-07 3.47e-07 3.39e-07 1.58595e-321 +gather: 3 (ozark): 3.5959e-05: breakdown: 2.109e-06 6.3e-07 3.87e-07 3.83e-07 3.79e-07 3.71e-07 4.03e-07 3.65e-07 3.66e-07 3.51e-07 3.48e-07 3.36e-07 3.41e-07 3.4e-07 3.4e-07 3.35e-07 3.34e-07 3.47e-07 3.41e-07 3.41e-07 3.35e-07 3.37e-07 3.38e-07 3.47e-07 3.4e-07 3.35e-07 3.38e-07 3.41e-07 3.49e-07 3.44e-07 3.34e-07 3.46e-07 3.42e-07 3.49e-07 3.42e-07 3.43e-07 3.35e-07 3.4e-07 3.4e-07 3.42e-07 3.4e-07 3.4e-07 3.44e-07 3.36e-07 3.36e-07 3.39e-07 3.49e-07 3.4e-07 3.36e-07 3.37e-07 3.37e-07 3.43e-07 3.36e-07 3.41e-07 3.45e-07 3.35e-07 3.46e-07 3.41e-07 3.47e-07 3.3e-07 3.33e-07 3.29e-07 3.36e-07 3.29e-07 3.37e-07 3.48e-07 3.55e-07 3.49e-07 3.39e-07 3.35e-07 3.43e-07 3.41e-07 3.39e-07 3.71e-07 3.4e-07 3.36e-07 3.32e-07 3.33e-07 3.33e-07 3.39e-07 3.34e-07 3.33e-07 3.3e-07 3.31e-07 3.36e-07 3.34e-07 3.31e-07 3.4e-07 3.31e-07 3.37e-07 3.4e-07 3.36e-07 3.39e-07 3.94e-07 3.33e-07 3.35e-07 3.3e-07 3.29e-07 3.27e-07 1.42785e-321 + +level2_complex +gather: 0 (ozark): 0.000867368: breakdown: 9.145e-06 8.915e-06 8.772e-06 8.76e-06 8.768e-06 8.72e-06 8.704e-06 8.679e-06 8.683e-06 8.686e-06 8.718e-06 8.67e-06 8.676e-06 8.661e-06 8.634e-06 8.686e-06 8.673e-06 8.63e-06 8.678e-06 8.659e-06 8.658e-06 8.706e-06 8.667e-06 8.679e-06 8.648e-06 8.653e-06 8.681e-06 8.668e-06 8.626e-06 8.665e-06 8.676e-06 8.659e-06 8.678e-06 8.721e-06 8.667e-06 8.638e-06 8.675e-06 8.742e-06 8.683e-06 8.698e-06 8.697e-06 8.687e-06 8.664e-06 8.672e-06 8.674e-06 8.689e-06 8.687e-06 8.671e-06 8.697e-06 8.724e-06 8.717e-06 8.711e-06 8.652e-06 8.673e-06 8.694e-06 8.702e-06 8.654e-06 8.661e-06 8.681e-06 8.687e-06 8.643e-06 8.712e-06 8.703e-06 8.643e-06 8.705e-06 8.67e-06 8.707e-06 8.654e-06 8.652e-06 8.72e-06 8.698e-06 8.676e-06 8.698e-06 8.713e-06 8.691e-06 8.744e-06 8.718e-06 8.691e-06 8.697e-06 8.709e-06 8.734e-06 8.669e-06 8.674e-06 8.694e-06 8.694e-06 8.635e-06 8.699e-06 8.694e-06 8.679e-06 8.667e-06 8.736e-06 8.684e-06 8.71e-06 8.697e-06 8.692e-06 8.701e-06 1.5241e-05 8.896e-06 8.629e-06 3.95747e-321 +gather: 1 (ozark): 0.000860855: breakdown: 9.106e-06 8.78e-06 8.755e-06 8.846e-06 8.663e-06 8.741e-06 8.653e-06 8.635e-06 8.666e-06 8.691e-06 8.608e-06 8.592e-06 8.711e-06 8.644e-06 8.571e-06 8.586e-06 8.654e-06 8.64e-06 8.55e-06 8.627e-06 8.656e-06 8.59e-06 8.635e-06 8.671e-06 8.584e-06 8.598e-06 8.646e-06 8.627e-06 8.597e-06 8.646e-06 8.615e-06 8.643e-06 8.589e-06 8.61e-06 8.627e-06 8.598e-06 8.589e-06 8.598e-06 8.625e-06 8.616e-06 8.58e-06 8.614e-06 8.611e-06 8.636e-06 8.597e-06 8.644e-06 8.629e-06 8.579e-06 8.68e-06 8.624e-06 8.615e-06 8.63e-06 8.625e-06 8.603e-06 8.65e-06 8.614e-06 8.63e-06 8.621e-06 8.618e-06 8.62e-06 8.651e-06 8.595e-06 8.625e-06 8.644e-06 8.623e-06 8.617e-06 8.631e-06 8.638e-06 8.58e-06 8.587e-06 8.651e-06 8.603e-06 8.565e-06 8.648e-06 8.621e-06 8.597e-06 8.617e-06 8.639e-06 8.629e-06 8.615e-06 8.588e-06 8.593e-06 8.601e-06 8.571e-06 8.602e-06 8.622e-06 8.59e-06 8.631e-06 8.654e-06 8.579e-06 8.589e-06 8.649e-06 8.628e-06 8.588e-06 8.587e-06 8.612e-06 8.575e-06 1.482e-05 8.901e-06 2.37646e-321 +gather: 2 (ozark): 0.000864709: breakdown: 8.987e-06 8.674e-06 8.645e-06 8.716e-06 8.76e-06 8.651e-06 8.673e-06 8.695e-06 8.67e-06 8.732e-06 8.645e-06 8.696e-06 8.743e-06 8.691e-06 8.679e-06 8.688e-06 8.682e-06 8.605e-06 8.626e-06 8.735e-06 8.702e-06 8.604e-06 8.672e-06 8.669e-06 8.716e-06 8.626e-06 8.684e-06 8.623e-06 8.592e-06 8.662e-06 8.67e-06 8.636e-06 8.63e-06 8.716e-06 8.669e-06 8.67e-06 8.649e-06 8.655e-06 8.649e-06 8.683e-06 8.69e-06 8.663e-06 8.676e-06 8.646e-06 8.702e-06 8.708e-06 8.636e-06 8.66e-06 8.739e-06 8.644e-06 8.669e-06 8.653e-06 8.649e-06 8.676e-06 8.652e-06 8.705e-06 8.695e-06 8.639e-06 8.679e-06 8.711e-06 8.675e-06 8.629e-06 8.69e-06 8.706e-06 8.638e-06 8.724e-06 8.67e-06 8.625e-06 8.662e-06 8.703e-06 8.672e-06 8.631e-06 8.68e-06 8.655e-06 8.655e-06 8.675e-06 8.651e-06 8.645e-06 8.677e-06 8.631e-06 8.656e-06 8.649e-06 8.672e-06 8.682e-06 8.639e-06 8.614e-06 8.628e-06 8.62e-06 8.681e-06 8.702e-06 8.663e-06 8.704e-06 8.722e-06 8.598e-06 8.653e-06 8.687e-06 1.4641e-05 8.854e-06 8.688e-06 2.45551e-321 +gather: 3 (ozark): 0.000864736: breakdown: 9.177e-06 8.76e-06 8.779e-06 8.695e-06 8.729e-06 8.753e-06 8.759e-06 8.691e-06 8.721e-06 8.666e-06 8.654e-06 8.587e-06 8.643e-06 8.645e-06 8.616e-06 8.648e-06 8.685e-06 8.63e-06 8.574e-06 8.721e-06 8.707e-06 8.637e-06 8.635e-06 8.636e-06 8.661e-06 8.712e-06 8.661e-06 8.628e-06 8.617e-06 8.63e-06 8.624e-06 8.713e-06 8.659e-06 8.655e-06 8.651e-06 8.689e-06 8.58e-06 8.691e-06 8.696e-06 8.583e-06 8.686e-06 8.65e-06 8.652e-06 8.624e-06 8.702e-06 8.657e-06 8.633e-06 8.644e-06 8.661e-06 8.669e-06 8.635e-06 8.668e-06 8.657e-06 8.663e-06 8.642e-06 8.648e-06 8.68e-06 8.659e-06 8.668e-06 8.623e-06 8.62e-06 8.649e-06 8.671e-06 8.666e-06 8.635e-06 8.721e-06 8.72e-06 8.652e-06 8.632e-06 8.694e-06 8.579e-06 8.617e-06 8.682e-06 8.633e-06 8.665e-06 8.673e-06 8.681e-06 8.67e-06 8.626e-06 8.644e-06 8.658e-06 8.68e-06 8.592e-06 8.689e-06 8.693e-06 8.642e-06 8.807e-06 8.682e-06 8.685e-06 8.687e-06 8.661e-06 8.68e-06 8.659e-06 8.618e-06 8.682e-06 8.731e-06 1.4807e-05 8.935e-06 8.699e-06 1.9812e-321 + +level3_complex +gather: 0 (ozark): 0.0538252: breakdown: 0.00055978 0.000543486 0.000548978 0.000548229 0.000538536 0.00054722 0.000543739 0.000546811 0.00053979 0.000550027 0.000541862 0.000547632 0.000543472 0.000545303 0.000549762 0.000537851 0.000549892 0.000544085 0.00054895 0.000540855 0.000548848 0.000540299 0.000544242 0.000545852 0.000549127 0.000548827 0.000535735 0.000547262 0.000539781 0.000546898 0.000543729 0.000547786 0.000540317 0.000545381 0.00054073 0.000546491 0.000550097 0.000535818 0.000547913 0.000542254 0.000547146 0.000539271 0.000545408 0.000541239 0.000543001 0.000539823 0.000547653 0.000539921 0.000539413 0.000546936 0.000542424 0.000548745 0.000539712 0.000545207 0.00053827 0.000543972 0.000539954 0.000543193 0.000537575 0.000543291 0.000545693 0.000536742 0.000546458 0.000540569 0.000545824 0.000540406 0.00054601 0.000539742 0.000545462 0.000539682 0.000546458 0.000546932 0.000537796 0.000546188 0.000541625 0.000547296 0.000540232 0.000546755 0.000540039 0.000547451 0.000539621 0.000546369 0.000539216 0.000529485 0.000546832 0.000540062 0.000547439 0.00054006 0.000545989 0.000539237 0.000542366 0.000539015 0.000547242 0.00054166 0.000542921 0.00054906 0.000535962 0.000549207 0.000540252 3.95747e-321 +gather: 1 (ozark): 0.0536701: breakdown: 0.000551591 0.000541947 0.000545451 0.000540808 0.000539546 0.000538551 0.000535682 0.00054463 0.000541579 0.000547235 0.00054549 0.000546029 0.00054306 0.00054624 0.000556297 0.000542625 0.000546403 0.000537593 0.000543726 0.000538994 0.000544698 0.000537991 0.000546843 0.000544523 0.000545878 0.00053929 0.000542762 0.000542064 0.000539548 0.000542867 0.00054179 0.000544508 0.000541368 0.000542865 0.000540428 0.000547096 0.000539224 0.000544651 0.000544216 0.000541226 0.000542148 0.000535662 0.000543039 0.000539396 0.000543314 0.000539253 0.000544487 0.000541574 0.000543724 0.000541306 0.000543121 0.000542549 0.000538428 0.00054271 0.000539764 0.000543492 0.000539096 0.000542387 0.000538867 0.000543126 0.000538717 0.000545024 0.00054263 0.000541388 0.000543049 0.000538711 0.000543948 0.000539657 0.00054413 0.00054032 0.000544341 0.000539652 0.000542949 0.00053813 0.000535218 0.000544154 0.000538467 0.000544665 0.000539001 0.000543345 0.000540691 0.000544732 0.000540622 0.000543947 0.000539468 0.000542463 0.000541501 0.000539309 0.000543069 0.000539667 0.000542574 0.000540279 0.000541807 0.000538951 0.000542046 0.000540192 0.000543317 0.000538987 0.000544213 4.79244e-322 +gather: 2 (ozark): 0.0538132: breakdown: 0.000550435 0.000543538 0.000543608 0.000544867 0.000542774 0.000545064 0.000541664 0.000545466 0.00054213 0.000548189 0.000546855 0.000554111 0.00055014 0.000559412 0.000565018 0.000545773 0.000552451 0.000543912 0.000545019 0.000539704 0.000548052 0.00054387 0.000552129 0.000549555 0.000548878 0.00054612 0.000542935 0.000546008 0.00054213 0.000545904 0.000541016 0.000545195 0.000539482 0.00054378 0.000539897 0.000542868 0.000546328 0.00054018 0.000544794 0.00054101 0.000543033 0.000539296 0.000544094 0.000538696 0.000544314 0.000540135 0.000542899 0.000546352 0.000543747 0.000544616 0.000539565 0.000543317 0.000539814 0.00054342 0.000539889 0.000543541 0.000540198 0.000543135 0.000540078 0.000544327 0.000545461 0.000539041 0.000541831 0.000539382 0.00054368 0.000538416 0.000543312 0.000539693 0.000544544 0.000540075 0.000543954 0.000544466 0.00053954 0.000546691 0.000541713 0.000547033 0.000539681 0.000543691 0.000539861 0.00054396 0.000540161 0.000542779 0.000542461 0.000538959 0.000542376 0.000538927 0.000541615 0.000538395 0.000543368 0.000538634 0.000542518 0.000539326 0.000541919 0.000538865 0.000543365 0.000544291 0.000541156 0.000544176 0.000539108 4.79244e-322 +gather: 3 (ozark): 0.0535708: breakdown: 0.000547376 0.000542838 0.000543107 0.000542089 0.000538403 0.000543558 0.000539586 0.000543657 0.000540671 0.000546131 0.00054811 0.000554863 0.000550522 0.000559185 0.000566595 0.000543105 0.000550561 0.000545429 0.000547224 0.000541117 0.000545292 0.000539994 0.000550631 0.000543468 0.000544843 0.000542488 0.000533926 0.000541398 0.000537941 0.000539308 0.00053623 0.000538794 0.00053757 0.000540321 0.000536797 0.000541441 0.000536323 0.000540603 0.000540146 0.000539231 0.00054142 0.000531915 0.000540206 0.000538299 0.000540316 0.000536559 0.000540995 0.000537689 0.000541769 0.000536105 0.000540842 0.000541265 0.000537981 0.000540714 0.000537244 0.000540758 0.000537241 0.000541429 0.000538617 0.000541695 0.000537148 0.000541746 0.000537491 0.000540748 0.000541341 0.000536701 0.00054033 0.000538023 0.0005418 0.000537335 0.000539515 0.000538278 0.000543444 0.000538244 0.000544648 0.000543962 0.000539196 0.000544652 0.000536673 0.000539884 0.000538555 0.000541914 0.000537178 0.000540089 0.00053624 0.000538286 0.000537264 0.000540487 0.000541481 0.000537355 0.00054157 0.000535785 0.000541459 0.000539439 0.000542353 0.000537141 0.000541068 0.000537104 0.000540902 4.79244e-322 + +level2_double +gather: 0 (ozark): 0.000378346: breakdown: 8.035e-06 3.892e-06 3.828e-06 3.807e-06 3.806e-06 3.811e-06 3.796e-06 3.784e-06 3.804e-06 3.777e-06 3.773e-06 3.791e-06 3.79e-06 3.75e-06 3.767e-06 3.774e-06 3.784e-06 3.793e-06 3.791e-06 3.765e-06 3.795e-06 3.776e-06 3.779e-06 3.766e-06 3.76e-06 3.789e-06 3.78e-06 3.762e-06 3.76e-06 3.774e-06 3.84e-06 3.807e-06 3.785e-06 3.788e-06 3.769e-06 3.757e-06 3.792e-06 3.79e-06 3.774e-06 3.807e-06 3.767e-06 3.77e-06 3.779e-06 3.75e-06 3.75e-06 3.76e-06 3.751e-06 3.77e-06 3.76e-06 3.781e-06 3.756e-06 3.776e-06 3.8e-06 3.796e-06 3.79e-06 3.783e-06 3.763e-06 3.786e-06 3.791e-06 3.744e-06 3.761e-06 3.755e-06 3.776e-06 3.754e-06 3.774e-06 3.755e-06 3.76e-06 3.757e-06 3.746e-06 3.771e-06 3.779e-06 3.753e-06 3.778e-06 3.749e-06 3.795e-06 3.826e-06 3.776e-06 3.744e-06 3.748e-06 3.781e-06 3.782e-06 3.773e-06 3.786e-06 3.776e-06 3.76e-06 3.759e-06 3.784e-06 3.798e-06 3.787e-06 3.782e-06 3.793e-06 3.771e-06 3.776e-06 3.742e-06 3.767e-06 3.777e-06 3.787e-06 3.812e-06 3.805e-06 4.79244e-322 +gather: 1 (ozark): 0.000377205: breakdown: 7.512e-06 3.92e-06 3.818e-06 3.831e-06 3.816e-06 3.802e-06 3.804e-06 3.793e-06 3.782e-06 3.817e-06 3.77e-06 3.767e-06 3.777e-06 3.8e-06 3.757e-06 3.766e-06 3.769e-06 3.796e-06 3.766e-06 3.754e-06 3.79e-06 3.754e-06 3.808e-06 3.78e-06 3.735e-06 3.778e-06 3.776e-06 3.766e-06 3.749e-06 3.779e-06 3.777e-06 3.77e-06 3.744e-06 3.791e-06 3.753e-06 3.794e-06 3.771e-06 3.729e-06 3.773e-06 3.763e-06 3.76e-06 3.774e-06 3.75e-06 3.754e-06 3.756e-06 3.755e-06 3.769e-06 3.778e-06 3.772e-06 3.742e-06 3.741e-06 3.778e-06 3.748e-06 3.851e-06 3.782e-06 3.741e-06 3.736e-06 3.777e-06 3.781e-06 3.765e-06 3.762e-06 3.772e-06 3.777e-06 3.787e-06 3.752e-06 3.772e-06 3.786e-06 3.761e-06 3.761e-06 3.809e-06 3.719e-06 3.765e-06 3.734e-06 3.81e-06 3.773e-06 3.752e-06 3.769e-06 3.789e-06 3.782e-06 3.746e-06 3.751e-06 3.778e-06 3.749e-06 3.769e-06 3.755e-06 3.786e-06 3.757e-06 3.777e-06 3.768e-06 3.769e-06 3.769e-06 3.769e-06 3.771e-06 3.762e-06 3.769e-06 3.758e-06 3.737e-06 3.752e-06 3.774e-06 4.79244e-322 +gather: 2 (ozark): 0.000375657: breakdown: 7.273e-06 3.933e-06 3.875e-06 3.833e-06 3.842e-06 3.807e-06 3.773e-06 3.807e-06 3.793e-06 3.766e-06 3.74e-06 3.735e-06 3.774e-06 3.789e-06 3.737e-06 3.752e-06 3.732e-06 3.796e-06 3.768e-06 3.755e-06 3.734e-06 3.734e-06 3.756e-06 3.768e-06 3.773e-06 3.754e-06 3.742e-06 3.75e-06 3.743e-06 3.727e-06 3.757e-06 3.781e-06 3.729e-06 3.754e-06 3.76e-06 3.755e-06 3.724e-06 3.766e-06 3.751e-06 3.728e-06 3.796e-06 3.748e-06 3.747e-06 3.735e-06 3.742e-06 3.782e-06 3.746e-06 3.751e-06 3.764e-06 3.754e-06 3.75e-06 3.755e-06 3.739e-06 3.748e-06 3.749e-06 3.751e-06 3.75e-06 3.779e-06 3.733e-06 3.727e-06 3.738e-06 3.728e-06 3.79e-06 3.781e-06 3.757e-06 3.749e-06 3.725e-06 3.738e-06 3.753e-06 3.738e-06 3.755e-06 3.751e-06 3.756e-06 3.766e-06 3.729e-06 3.771e-06 3.748e-06 3.742e-06 3.748e-06 3.737e-06 3.763e-06 3.746e-06 3.758e-06 3.763e-06 3.738e-06 3.714e-06 3.743e-06 3.75e-06 3.779e-06 3.761e-06 3.739e-06 3.782e-06 3.735e-06 3.732e-06 3.785e-06 3.781e-06 3.765e-06 3.773e-06 3.738e-06 4.79244e-322 +gather: 3 (ozark): 0.000377028: breakdown: 6.824e-06 3.871e-06 3.843e-06 3.854e-06 3.794e-06 3.767e-06 3.777e-06 3.774e-06 3.775e-06 3.784e-06 3.82e-06 3.761e-06 3.798e-06 3.75e-06 3.768e-06 3.767e-06 3.779e-06 3.788e-06 3.797e-06 3.783e-06 3.823e-06 3.782e-06 3.818e-06 3.782e-06 3.803e-06 3.748e-06 3.767e-06 3.797e-06 3.798e-06 3.713e-06 3.788e-06 3.789e-06 3.737e-06 3.807e-06 3.792e-06 3.732e-06 3.774e-06 3.852e-06 3.785e-06 3.759e-06 3.756e-06 3.753e-06 3.79e-06 3.761e-06 3.749e-06 3.799e-06 3.757e-06 3.769e-06 3.772e-06 3.748e-06 3.805e-06 3.778e-06 3.747e-06 3.738e-06 3.755e-06 3.742e-06 3.797e-06 3.78e-06 3.756e-06 3.757e-06 3.798e-06 3.779e-06 3.762e-06 3.759e-06 3.763e-06 3.813e-06 3.798e-06 3.773e-06 3.767e-06 3.752e-06 3.784e-06 3.742e-06 3.782e-06 3.807e-06 3.798e-06 3.759e-06 3.8e-06 3.738e-06 3.759e-06 3.768e-06 3.781e-06 3.813e-06 3.803e-06 3.752e-06 3.794e-06 3.721e-06 3.749e-06 3.785e-06 3.8e-06 3.785e-06 3.766e-06 3.756e-06 3.771e-06 3.735e-06 3.765e-06 3.767e-06 3.779e-06 3.767e-06 3.809e-06 4.79244e-322 + +level3_double +gather: 0 (ozark): 0.013612: breakdown: 0.000137439 0.000135248 0.000142266 0.00013641 0.00013666 0.000136452 0.000136549 0.000137653 0.000136242 0.000136384 0.000139447 0.000135525 0.000135817 0.000136879 0.000136827 0.000136968 0.0001368 0.000151418 0.000136682 0.000136878 0.000136249 0.00013699 0.000138251 0.000138002 0.000144697 0.000138951 0.000138674 0.000139006 0.000138346 0.000138737 0.000138277 0.000150323 0.000135822 0.00013623 0.000136439 0.000135897 0.000136263 0.00013613 0.000142 0.000134647 0.000135762 0.000135923 0.000135727 0.000135732 0.000135398 0.000136092 0.000140582 0.000135843 0.000136975 0.000136878 0.000136689 0.000137124 0.000137116 0.000142684 0.000135559 0.000136155 0.000136384 0.000136855 0.000137057 0.000137618 0.000142347 0.000135631 0.000136813 0.000137354 0.000136669 0.000137349 0.000136907 0.000136542 0.000143874 0.000135867 0.000135643 0.000136666 0.000136657 0.000136491 0.000136529 0.000139722 0.000136014 0.000136526 0.000136317 0.000136224 0.000136852 0.000135923 0.000142899 0.000135799 0.000135893 0.000136045 0.000136262 0.000136381 0.000136772 0.000140402 0.000135093 0.000135558 0.000135691 0.000135739 0.000135598 0.000135852 0.000135737 0.000145457 0.000135248 7.95446e-322 +gather: 1 (ozark): 0.1136195: breakdown: 0.000138176 0.00013565 0.000143277 0.000136673 0.000136618 0.00013671 0.000136653 0.000136514 0.000137308 0.000136652 0.000142868 0.0001366 0.000137026 0.000136268 0.000136929 0.000136964 0.000138146 0.000142308 0.0001375 0.000137346 0.000137585 0.000138227 0.000137883 0.000137738 0.000142911 0.000138691 0.000137783 0.00013784 0.000138889 0.000138536 0.000138032 0.000143095 0.000136961 0.000137316 0.000137544 0.000137264 0.000136793 0.000136994 0.000140623 0.000137524 0.000136353 0.000137041 0.000136458 0.000137129 0.000136572 0.000137049 0.000140948 0.000134394 0.000135512 0.000136655 0.000136194 0.00013667 0.000137126 0.000141529 0.000137369 0.000136999 0.000136428 0.000136381 0.000136402 0.000136151 0.000141558 0.000134545 0.000136237 0.000136804 0.000136795 0.000137166 0.000137481 0.000137738 0.000141553 0.000137137 0.000136745 0.000136077 0.00013621 0.000136482 0.000136231 0.000140824 0.000136197 0.000136674 0.000135973 0.000136704 0.000136847 0.000136922 0.000141772 0.00013803 0.000136442 0.000136615 0.000136524 0.000136912 0.000136511 0.0001413 0.000134221 0.000136172 0.000136573 0.000136899 0.000136804 0.000137158 0.000136199 0.000143031 0.000135647 4.79244e-322 +gather: 2 (ozark): 0.2136247: breakdown: 0.00013855 0.000136033 0.000141318 0.000136556 0.00013614 0.000137407 0.000137245 0.000137249 0.000136647 0.000137335 0.000142316 0.000137039 0.000136559 0.00013707 0.000137341 0.000137757 0.000137499 0.000143004 0.000137948 0.000138045 0.00013743 0.000137632 0.000137593 0.000137794 0.000142373 0.000138192 0.000137964 0.000139087 0.000138493 0.000138635 0.000138871 0.000142191 0.000138046 0.0001377 0.000137257 0.00013722 0.000137161 0.000136117 0.000142166 0.000136582 0.000136953 0.000137206 0.000137419 0.000137067 0.00013683 0.000136755 0.000141894 0.00013716 0.000136754 0.000136251 0.000136464 0.000136613 0.000137158 0.000141419 0.000137366 0.000136705 0.000136541 0.000137122 0.000136445 0.000137042 0.000141469 0.000136135 0.000137362 0.000137254 0.000136432 0.000137583 0.000136443 0.000137165 0.000139909 0.000135689 0.000137449 0.000136929 0.00013699 0.000136446 0.000137062 0.000138618 0.000136036 0.000136417 0.000137072 0.00013672 0.000137436 0.000137476 0.000139347 0.00013521 0.000136924 0.000136824 0.000137233 0.000136498 0.000136275 0.000139838 0.000136246 0.000136318 0.000136209 0.00013724 0.000136101 0.000136778 0.000135724 0.00014169 0.00013543 4.79244e-322 +gather: 3 (ozark): 0.0136833: breakdown: 0.000137755 0.000137057 0.000161103 0.000137187 0.000137078 0.000136033 0.000136746 0.000136884 0.00013684 0.000141032 0.000136113 0.000136183 0.000136757 0.000137477 0.000137461 0.000137472 0.000137179 0.000175561 0.000137687 0.000137406 0.00013739 0.000137872 0.000137693 0.000138476 0.000142229 0.00013853 0.000138437 0.000138516 0.000139438 0.000138633 0.000138144 0.000142989 0.000139092 0.000137896 0.0001369 0.000137677 0.000137364 0.000136515 0.00014102 0.000137012 0.000136586 0.00013587 0.000136078 0.000136019 0.000136107 0.000141825 0.000136524 0.000136417 0.000135826 0.000136311 0.000136288 0.000136025 0.000136041 0.000150565 0.000136578 0.000136137 0.000136654 0.000136913 0.000137091 0.000136599 0.000141485 0.00013705 0.000136881 0.000136513 0.000137141 0.000136549 0.00013709 0.000143895 0.000136945 0.000136209 0.000136313 0.000136171 0.000136267 0.000136307 0.000140881 0.000135192 0.000136107 0.000135952 0.000136634 0.000136678 0.000136549 0.000136721 0.000141329 0.000137277 0.000136741 0.000136888 0.000136578 0.000136789 0.000136383 0.0001411 0.000137554 0.000136821 0.000137963 0.000135352 0.0001367 0.000136426 0.000141386 0.000135826 0.000137355 4.79244e-322 + +level1_complex +gather: 0 (ozark): 1.8592e-05: breakdown: 4.16e-07 2.55e-07 2.24e-07 2.26e-07 2.23e-07 2.32e-07 2.11e-07 2.02e-07 1.83e-07 2.15e-07 2.01e-07 1.83e-07 1.83e-07 1.83e-07 1.86e-07 1.82e-07 1.82e-07 1.81e-07 1.82e-07 1.83e-07 1.81e-07 1.81e-07 1.81e-07 1.82e-07 1.83e-07 1.83e-07 1.84e-07 1.83e-07 1.81e-07 1.8e-07 1.82e-07 1.81e-07 1.81e-07 1.81e-07 1.81e-07 1.81e-07 1.82e-07 2.31e-07 1.81e-07 1.82e-07 1.8e-07 1.8e-07 1.8e-07 1.84e-07 1.82e-07 1.8e-07 1.82e-07 1.81e-07 1.81e-07 1.8e-07 1.81e-07 1.8e-07 1.81e-07 1.81e-07 1.8e-07 1.81e-07 1.8e-07 1.82e-07 1.81e-07 1.79e-07 1.83e-07 1.8e-07 1.83e-07 1.8e-07 1.82e-07 1.8e-07 1.81e-07 1.79e-07 1.81e-07 1.82e-07 1.8e-07 1.82e-07 1.81e-07 1.8e-07 1.82e-07 1.8e-07 1.79e-07 1.81e-07 1.82e-07 1.82e-07 1.82e-07 1.82e-07 1.8e-07 1.83e-07 1.81e-07 1.8e-07 1.81e-07 1.8e-07 1.81e-07 1.81e-07 1.82e-07 1.8e-07 1.8e-07 1.82e-07 1.81e-07 1.81e-07 1.81e-07 1.82e-07 1.81e-07 4.79244e-322 +gather: 1 (ozark): 2.1363e-05: breakdown: 4.54e-07 2.88e-07 2.6e-07 2.71e-07 2.59e-07 2.53e-07 2.43e-07 2.35e-07 2.34e-07 2.2e-07 3e-07 2.58e-07 2.11e-07 2.12e-07 2.13e-07 2.1e-07 2.11e-07 2.09e-07 2.09e-07 2.1e-07 2.1e-07 2.08e-07 2.07e-07 2.07e-07 2.07e-07 2.07e-07 2.08e-07 2.07e-07 2.06e-07 2.08e-07 2.08e-07 2.06e-07 2.07e-07 2.08e-07 2.06e-07 2.07e-07 2.08e-07 2.06e-07 2.07e-07 2.05e-07 2.09e-07 2.09e-07 2.08e-07 2.08e-07 2.08e-07 2.05e-07 2.07e-07 2.07e-07 2.06e-07 2.05e-07 2.07e-07 2.07e-07 2.06e-07 2.1e-07 2.06e-07 2.06e-07 2.09e-07 2.08e-07 2.08e-07 2.06e-07 2.07e-07 2.09e-07 2.05e-07 2.08e-07 2.09e-07 2.09e-07 2.07e-07 2.06e-07 2.08e-07 2.07e-07 2.08e-07 2.08e-07 2.05e-07 2.09e-07 2.06e-07 2.07e-07 2.06e-07 2.07e-07 2.06e-07 2.06e-07 2.07e-07 2.06e-07 2.06e-07 2.07e-07 2.06e-07 2.5e-07 2.06e-07 2.08e-07 2.09e-07 2.06e-07 2.06e-07 2.08e-07 2.07e-07 2.11e-07 2.06e-07 2.04e-07 2.07e-07 2.04e-07 2.08e-07 4.79244e-322 +gather: 2 (ozark): 1.9089e-05: breakdown: 4.77e-07 3.16e-07 2.44e-07 2.26e-07 2.44e-07 2.19e-07 2.55e-07 2.03e-07 2.04e-07 1.97e-07 1.87e-07 1.82e-07 1.86e-07 1.81e-07 1.83e-07 1.86e-07 1.88e-07 1.86e-07 2.21e-07 1.82e-07 1.84e-07 1.84e-07 1.81e-07 1.82e-07 1.84e-07 1.84e-07 1.84e-07 1.84e-07 1.83e-07 1.83e-07 1.84e-07 1.82e-07 1.83e-07 1.83e-07 1.82e-07 1.85e-07 1.91e-07 1.83e-07 1.88e-07 1.84e-07 1.82e-07 1.87e-07 1.83e-07 1.8e-07 1.87e-07 1.82e-07 1.85e-07 1.87e-07 1.83e-07 1.85e-07 1.85e-07 1.82e-07 1.83e-07 2.47e-07 1.84e-07 1.82e-07 1.86e-07 1.83e-07 1.85e-07 1.85e-07 1.8e-07 1.84e-07 1.83e-07 1.83e-07 1.83e-07 1.86e-07 1.85e-07 1.84e-07 1.86e-07 1.85e-07 1.85e-07 1.85e-07 1.85e-07 1.85e-07 1.86e-07 1.84e-07 1.83e-07 1.84e-07 1.83e-07 1.85e-07 1.86e-07 1.85e-07 1.84e-07 1.85e-07 1.83e-07 1.86e-07 1.89e-07 1.85e-07 1.86e-07 1.83e-07 1.86e-07 1.85e-07 1.84e-07 1.85e-07 1.86e-07 1.84e-07 1.86e-07 1.83e-07 1.86e-07 4.79244e-322 +gather: 3 (ozark): 1.8519e-05: breakdown: 4.53e-07 2.44e-07 2.29e-07 2.29e-07 2.26e-07 2.15e-07 1.97e-07 1.82e-07 1.85e-07 1.8e-07 1.83e-07 1.79e-07 1.81e-07 1.8e-07 1.84e-07 1.8e-07 1.79e-07 1.79e-07 1.77e-07 1.8e-07 1.8e-07 1.84e-07 1.8e-07 1.81e-07 1.8e-07 1.81e-07 1.82e-07 1.8e-07 1.81e-07 1.79e-07 1.8e-07 1.8e-07 1.79e-07 1.8e-07 1.8e-07 1.82e-07 1.82e-07 1.8e-07 1.8e-07 1.8e-07 1.79e-07 1.81e-07 1.82e-07 1.81e-07 1.82e-07 1.8e-07 1.8e-07 1.79e-07 1.79e-07 1.8e-07 1.79e-07 1.82e-07 1.83e-07 1.79e-07 1.86e-07 1.79e-07 1.8e-07 1.82e-07 1.82e-07 1.83e-07 1.82e-07 1.82e-07 1.79e-07 1.82e-07 1.78e-07 1.81e-07 1.78e-07 1.79e-07 1.8e-07 1.8e-07 1.78e-07 2.41e-07 1.8e-07 1.8e-07 1.8e-07 1.81e-07 1.83e-07 1.79e-07 1.81e-07 1.84e-07 1.81e-07 1.78e-07 1.78e-07 1.8e-07 1.83e-07 1.82e-07 1.8e-07 1.79e-07 1.8e-07 1.79e-07 1.79e-07 1.82e-07 1.81e-07 1.8e-07 1.8e-07 1.8e-07 1.81e-07 2.38e-07 1.79e-07 4.79244e-322 + +level1_double +gather: 0 (ozark): 2.6729e-05: breakdown: 1.212e-06 3.51e-07 3.53e-07 3.26e-07 3.21e-07 2.95e-07 2.81e-07 3.3e-07 2.77e-07 3.08e-07 2.66e-07 2.67e-07 2.89e-07 2.58e-07 2.58e-07 2.92e-07 2.49e-07 3.38e-07 2.67e-07 2.56e-07 2.77e-07 2.55e-07 2.55e-07 3.01e-07 2.43e-07 2.43e-07 2.35e-07 2.57e-07 2.64e-07 2.48e-07 2.42e-07 2.48e-07 2.88e-07 2.43e-07 2.39e-07 2.47e-07 2.53e-07 2.46e-07 2.45e-07 2.39e-07 2.88e-07 2.99e-07 2.38e-07 2.41e-07 2.46e-07 2.76e-07 2.43e-07 2.46e-07 2.42e-07 2.42e-07 2.76e-07 2.41e-07 2.45e-07 2.53e-07 2.32e-07 2.32e-07 2.4e-07 2.28e-07 2.38e-07 2.78e-07 2.65e-07 2.4e-07 2.41e-07 2.46e-07 2.58e-07 2.66e-07 2.62e-07 2.96e-07 2.5e-07 2.45e-07 2.46e-07 2.66e-07 2.45e-07 2.35e-07 2.33e-07 2.33e-07 2.97e-07 2.42e-07 2.38e-07 2.49e-07 2.55e-07 2.61e-07 2.39e-07 2.41e-07 2.43e-07 2.99e-07 2.48e-07 2.41e-07 2.46e-07 2.61e-07 2.63e-07 2.44e-07 2.33e-07 2.42e-07 2.78e-07 2.49e-07 2.37e-07 2.54e-07 2.56e-07 1.58941e-320 +gather: 1 (ozark): 2.3465e-05: breakdown: 1.414e-06 3.56e-07 3.11e-07 2.95e-07 2.71e-07 2.64e-07 2.56e-07 2.45e-07 2.34e-07 2.31e-07 2.29e-07 2.26e-07 2.29e-07 2.23e-07 2.29e-07 2.25e-07 2.23e-07 2.25e-07 2.85e-07 2.19e-07 2.19e-07 2.22e-07 2.87e-07 2.18e-07 2.2e-07 2.22e-07 2.19e-07 2.2e-07 2.29e-07 2.23e-07 2.24e-07 2.18e-07 2.25e-07 2.19e-07 2.21e-07 2.18e-07 2.23e-07 2.2e-07 2.17e-07 2.28e-07 2.19e-07 2.19e-07 2.2e-07 2.14e-07 2.16e-07 2.18e-07 2.14e-07 2.13e-07 2.13e-07 2.13e-07 2.15e-07 2.17e-07 2.13e-07 2.14e-07 2.16e-07 2.19e-07 2.15e-07 2.19e-07 2.15e-07 2.12e-07 2.15e-07 2.14e-07 2.2e-07 2.19e-07 2.2e-07 2.14e-07 2.13e-07 2.14e-07 2.17e-07 2.13e-07 2.11e-07 2.15e-07 2.15e-07 2.17e-07 2.19e-07 2.12e-07 2.13e-07 2.16e-07 2.14e-07 2.18e-07 2.82e-07 2.16e-07 2.21e-07 2.13e-07 2.13e-07 2.15e-07 2.13e-07 2.15e-07 2.17e-07 2.14e-07 2.15e-07 2.17e-07 2.12e-07 2.17e-07 2.13e-07 2.16e-07 2.19e-07 2.16e-07 2.16e-07 1.58941e-320 +gather: 2 (ozark): 2.35e-05: breakdown: 1.522e-06 3.73e-07 3.35e-07 2.75e-07 3.24e-07 3.07e-07 2.99e-07 2.41e-07 2.4e-07 2.4e-07 2.41e-07 2.18e-07 2.18e-07 2.19e-07 2.21e-07 2.18e-07 2.2e-07 2.11e-07 2.14e-07 2.17e-07 2.17e-07 2.2e-07 2.14e-07 2.18e-07 2.15e-07 2.18e-07 2.13e-07 2.15e-07 2.12e-07 2.12e-07 2.16e-07 2.15e-07 2.14e-07 2.21e-07 2.12e-07 2.14e-07 2.19e-07 2.12e-07 2.15e-07 2.15e-07 2.12e-07 2.14e-07 2.1e-07 2.12e-07 2.61e-07 2.14e-07 2.13e-07 2.37e-07 2.15e-07 2.2e-07 2.18e-07 2.18e-07 2.19e-07 2.2e-07 2.17e-07 2.2e-07 2.18e-07 2.18e-07 2.2e-07 2.17e-07 2.18e-07 2.16e-07 2.15e-07 2.13e-07 2.14e-07 2.18e-07 2.15e-07 2.15e-07 2.13e-07 2.11e-07 2.12e-07 2.1e-07 2.14e-07 2.13e-07 2.13e-07 2.16e-07 2.17e-07 2.12e-07 2.16e-07 2.13e-07 2.14e-07 2.13e-07 2.13e-07 2.14e-07 2.11e-07 2.13e-07 2.14e-07 2.11e-07 2.12e-07 2.16e-07 2.1e-07 3.21e-07 2.14e-07 2.14e-07 2.16e-07 2.1e-07 2.17e-07 2.13e-07 2.17e-07 1.58941e-320 +gather: 3 (ozark): 2.3176e-05: breakdown: 1.772e-06 3.98e-07 2.99e-07 3.59e-07 2.62e-07 2.66e-07 2.55e-07 2.35e-07 2.24e-07 2.32e-07 2.2e-07 2.16e-07 2.22e-07 2.19e-07 2.16e-07 2.16e-07 2.16e-07 2.09e-07 2.31e-07 2.15e-07 2.12e-07 2.15e-07 2.13e-07 2.14e-07 2.13e-07 2.11e-07 2.14e-07 2.11e-07 2.12e-07 2.11e-07 2.15e-07 2.1e-07 2.13e-07 2.03e-07 2.03e-07 2.14e-07 2.1e-07 2.14e-07 2.11e-07 2.11e-07 2.15e-07 2.16e-07 2.12e-07 2.15e-07 2.11e-07 2.09e-07 2.1e-07 2.12e-07 2.1e-07 2.12e-07 2.08e-07 2.72e-07 2.08e-07 2.08e-07 2.09e-07 2.07e-07 2.05e-07 2.09e-07 2.08e-07 2.06e-07 2.09e-07 2.09e-07 2.09e-07 2.11e-07 2.12e-07 2.1e-07 2.07e-07 2.08e-07 2.1e-07 2.08e-07 2.11e-07 2.1e-07 2.09e-07 2.11e-07 2.12e-07 2.11e-07 2.11e-07 2.13e-07 2.11e-07 2.1e-07 2.1e-07 2.11e-07 2.1e-07 2.1e-07 2.09e-07 2.08e-07 2.1e-07 2.12e-07 2.12e-07 2.1e-07 2.14e-07 2.08e-07 2.1e-07 2.12e-07 2.1e-07 2.12e-07 2.12e-07 2.1e-07 2.1e-07 1.58941e-320 diff --git a/tests/unit/detection/test_slow_node_detector.py b/tests/unit/detection/test_slow_node_detector.py index f0b25aa..545a07e 100644 --- a/tests/unit/detection/test_slow_node_detector.py +++ b/tests/unit/detection/test_slow_node_detector.py @@ -18,7 +18,7 @@ def setUp(self): self.spn = 1 # Determine expected values - self.expected_slow_ranks = [1, 2, 4, 6, 7, 9, 10, 11] + self.expected_slow_ranks = [1, 2] self.expected_slow_nodes = [] self.expected_overheated_nodes = ["ozark"] @@ -28,6 +28,8 @@ def setUp(self): sensors=self.sensors_file, num_nodes=2, pct=0.05, + benchmark="level3", + type="double", spn=self.spn, rpn=self.rpn, plot_rank_breakdowns=False, From 1e2e81b68d8490d4f040d49d8063f1d3dddda52c Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Thu, 12 Jun 2025 08:23:47 -0400 Subject: [PATCH 02/10] #10: allow for different N for each benchmark --- src/benchmarks.cc | 14 ++++++++------ src/benchmarks.h | 2 +- src/slow_node.cc | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/benchmarks.cc b/src/benchmarks.cc index 834726c..dbad98f 100644 --- a/src/benchmarks.cc +++ b/src/benchmarks.cc @@ -11,7 +11,7 @@ namespace benchmarks { template <> std::string typeToString() { - return "double"; + return "double "; } template <> @@ -61,7 +61,7 @@ benchmark_results_t runBenchmarkLevel1(int N, int iters) { int rank = -1; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - std::cout << "[level1 " << typeToString() << "] rank: " << rank << ", total_time=" << total_time << std::endl; + std::cout << "[level1 " << typeToString() << "] (N=" << N << ") rank: " << rank << ", total_time=" << total_time << std::endl; return std::make_tuple(iter_timings, total_time); } @@ -101,7 +101,7 @@ benchmark_results_t runBenchmarkLevel2(int M, int N, int iters) { int rank = -1; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - std::cout << "[level2 " << typeToString() << "] rank: " << rank << ", total_time=" << total_time << std::endl; + std::cout << "[level2 " << typeToString() << "] (M=" << M << ", N=" << N << ") rank: " << rank << ", total_time=" << total_time << std::endl; return std::make_tuple(iter_timings, total_time); } @@ -142,7 +142,7 @@ benchmark_results_t runBenchmarkLevel3(int M, int N, int K, int iters) { int rank = -1; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - std::cout << "[level3 " << typeToString() << "] rank: " << rank << ", total_time=" << total_time << std::endl; + std::cout << "[level3 " << typeToString() << "] (M=" << M << ", N=" << N << ", K=" << K << ") rank: " << rank << ", total_time=" << total_time << std::endl; return std::make_tuple(iter_timings, total_time); } @@ -191,7 +191,7 @@ benchmark_results_t runBenchmarkDPOTRF(int N, int iters) { int rank = -1; MPI_Comm_rank(MPI_COMM_WORLD, &rank); - std::cout << "[dpotrf " << typeToString() << "] rank: " << rank << ", total_time=" << total_time << std::endl; + std::cout << "[dpotrf " << typeToString() << "] (N=" << N << ") rank: " << rank << ", total_time=" << total_time << std::endl; return std::make_tuple(iter_timings, total_time); } @@ -212,9 +212,11 @@ benchmark_results_t runBenchmark(benchmark_types b, int M, int N, int K, int ite } } -all_results_t runAllBenchmarks(int M, int N, int K, int iters) { +all_results_t runAllBenchmarks(int M, int N1, int N2, int N3, int K, int iters) { all_results_t all_results; + int N; for (int i=0; i < benchmark_types::num_benchmarks; i++) { + N = i == 0 ? N1 : (i == 1 ? N2 : N3); auto b = static_cast(i); std::string benchmark_str = benchmarkToString(b); all_results[benchmark_str + "_double"] = runBenchmark(b, M, N, K, iters); diff --git a/src/benchmarks.h b/src/benchmarks.h index 8591df8..a535d8d 100644 --- a/src/benchmarks.h +++ b/src/benchmarks.h @@ -37,7 +37,7 @@ benchmark_results_t runBenchmarkDPOTRF(int N, int iters); template benchmark_results_t runBenchmark(benchmark_types b, int M, int N, int K, int iters); -all_results_t runAllBenchmarks(int M, int N, int K, int iters); +all_results_t runAllBenchmarks(int M, int N1, int N2, int N3, int K, int iters); void printBenchmarkOutput(all_results_t benchmark_results, int iters); diff --git a/src/slow_node.cc b/src/slow_node.cc index 8206aea..e434f75 100644 --- a/src/slow_node.cc +++ b/src/slow_node.cc @@ -8,15 +8,41 @@ static int iters = 100; static int M = 128; -static int N = 128; +static int N1 = 128; +static int N2 = 128; +static int N3 = 128; static int K = 128; +/* + * USAGE: ./slow_node | + * + * M and K share the same value, given by the slot. + * + * If and are not provided, the value given for + * M and K is also used for all N. + * + * Similarly, if is not provided, the value given for + * is used for all N, and so on with . + */ + int main(int argc, char** argv) { + if (argc > 1) { iters = atoi(argv[1]) + 1; // add one iteration since we will drop the first one - M = N = K = atoi(argv[2]); + M = N1 = N2 = N3 = K = atoi(argv[2]); + if (argc > 3) { + N1 = N2 = N3 = atoi(argv[3]); + if (argc > 4) { + N2 = N3 = atoi(argv[4]); + if (argc > 5) { + N3 = atoi(argv[5]); + } + } + } } - std::cout << "iters: " << iters << ", M=N=K=" << M << std::endl; + + std::cout << "iters: " << iters << ", M=K=" << M << std::endl; + std::cout << "N1=" << N1 << ", N2=" << N2 << ", N3=" << N3 << std::endl; MPI_Init(&argc, &argv); Kokkos::initialize(argc, argv); @@ -27,7 +53,7 @@ int main(int argc, char** argv) { // Loop through all available benchmarks sensors::runSensorsAndReduceOutput(processor_name, "pre"); - auto output = benchmarks::runAllBenchmarks(M, N, K, iters); + auto output = benchmarks::runAllBenchmarks(M, N1, N2, N3, K, iters); sensors::runSensorsAndReduceOutput(processor_name, "post"); benchmarks::printBenchmarkOutput(output, iters); From 1d2d4bce0a831782e384fedc4346d4c66df9c3ef Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Thu, 12 Jun 2025 09:04:40 -0400 Subject: [PATCH 03/10] #10: fix error with last iteration timing --- src/slow_node.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/slow_node.cc b/src/slow_node.cc index e434f75..2f5dc52 100644 --- a/src/slow_node.cc +++ b/src/slow_node.cc @@ -28,7 +28,7 @@ static int K = 128; int main(int argc, char** argv) { if (argc > 1) { - iters = atoi(argv[1]) + 1; // add one iteration since we will drop the first one + iters = atoi(argv[1]); M = N1 = N2 = N3 = K = atoi(argv[2]); if (argc > 3) { N1 = N2 = N3 = atoi(argv[3]); @@ -53,7 +53,7 @@ int main(int argc, char** argv) { // Loop through all available benchmarks sensors::runSensorsAndReduceOutput(processor_name, "pre"); - auto output = benchmarks::runAllBenchmarks(M, N1, N2, N3, K, iters); + auto output = benchmarks::runAllBenchmarks(M, N1, N2, N3, K, iters + 1); // add one iteration since we'll drop the first one sensors::runSensorsAndReduceOutput(processor_name, "post"); benchmarks::printBenchmarkOutput(output, iters); From 314ce6c3349990957f1aa6a5bd64bac8e619d6f1 Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Thu, 12 Jun 2025 14:16:05 -0400 Subject: [PATCH 04/10] #10: allow user to set all matrix sizes for all benchmarks --- src/benchmarks.cc | 18 ++++++++---------- src/benchmarks.h | 4 ++-- src/slow_node.cc | 41 +++++++++++++++-------------------------- 3 files changed, 25 insertions(+), 38 deletions(-) diff --git a/src/benchmarks.cc b/src/benchmarks.cc index dbad98f..116d35b 100644 --- a/src/benchmarks.cc +++ b/src/benchmarks.cc @@ -197,30 +197,28 @@ benchmark_results_t runBenchmarkDPOTRF(int N, int iters) { } template -benchmark_results_t runBenchmark(benchmark_types b, int M, int N, int K, int iters) { +benchmark_results_t runBenchmark(benchmark_types b, std::vector sizes, int iters) { switch (b) { case level1: - return runBenchmarkLevel1(N, iters); + return runBenchmarkLevel1(sizes[0], iters); case level2: - return runBenchmarkLevel2(M, N, iters); + return runBenchmarkLevel2(sizes[1], sizes[2], iters); case level3: - return runBenchmarkLevel3(M, N, K, iters); + return runBenchmarkLevel3(sizes[3], sizes[4], sizes[5], iters); case dpotrf: - return runBenchmarkDPOTRF(N, iters); + return runBenchmarkDPOTRF(sizes[6], iters); default: throw std::invalid_argument("Unsupported benchmark type"); } } -all_results_t runAllBenchmarks(int M, int N1, int N2, int N3, int K, int iters) { +all_results_t runAllBenchmarks(std::vector sizes, int iters) { all_results_t all_results; - int N; for (int i=0; i < benchmark_types::num_benchmarks; i++) { - N = i == 0 ? N1 : (i == 1 ? N2 : N3); auto b = static_cast(i); std::string benchmark_str = benchmarkToString(b); - all_results[benchmark_str + "_double"] = runBenchmark(b, M, N, K, iters); - all_results[benchmark_str + "_complex"] = runBenchmark>(b, M, N, K, iters); + all_results[benchmark_str + "_double"] = runBenchmark(b, sizes, iters); + all_results[benchmark_str + "_complex"] = runBenchmark>(b, sizes, iters); } return all_results; } diff --git a/src/benchmarks.h b/src/benchmarks.h index a535d8d..eeb004f 100644 --- a/src/benchmarks.h +++ b/src/benchmarks.h @@ -35,9 +35,9 @@ template benchmark_results_t runBenchmarkDPOTRF(int N, int iters); template -benchmark_results_t runBenchmark(benchmark_types b, int M, int N, int K, int iters); +benchmark_results_t runBenchmark(benchmark_types b, std::vector sizes, int iters); -all_results_t runAllBenchmarks(int M, int N1, int N2, int N3, int K, int iters); +all_results_t runAllBenchmarks(std::vector sizes, int iters); void printBenchmarkOutput(all_results_t benchmark_results, int iters); diff --git a/src/slow_node.cc b/src/slow_node.cc index 2f5dc52..1b48ad6 100644 --- a/src/slow_node.cc +++ b/src/slow_node.cc @@ -7,43 +7,32 @@ #include static int iters = 100; -static int M = 128; static int N1 = 128; +static int M2 = 128; static int N2 = 128; +static int M3 = 128; static int N3 = 128; -static int K = 128; +static int K3 = 128; +static int N4 = 128; /* - * USAGE: ./slow_node | - * - * M and K share the same value, given by the slot. - * - * If and are not provided, the value given for - * M and K is also used for all N. - * - * Similarly, if is not provided, the value given for - * is used for all N, and so on with . + * USAGE: ./slow_node */ - int main(int argc, char** argv) { + std::vector sizes; + if (argc > 1) { iters = atoi(argv[1]); - M = N1 = N2 = N3 = K = atoi(argv[2]); - if (argc > 3) { - N1 = N2 = N3 = atoi(argv[3]); - if (argc > 4) { - N2 = N3 = atoi(argv[4]); - if (argc > 5) { - N3 = atoi(argv[5]); - } - } - } + sizes.push_back(atoi(argv[2])); /* N1 */ + sizes.push_back(atoi(argv[3])); /* M2 */ + sizes.push_back(atoi(argv[4])); /* N2 */ + sizes.push_back(atoi(argv[5])); /* M3 */ + sizes.push_back(atoi(argv[6])); /* N3 */ + sizes.push_back(atoi(argv[7])); /* K3 */ + sizes.push_back(atoi(argv[8])); /* N4 */ } - std::cout << "iters: " << iters << ", M=K=" << M << std::endl; - std::cout << "N1=" << N1 << ", N2=" << N2 << ", N3=" << N3 << std::endl; - MPI_Init(&argc, &argv); Kokkos::initialize(argc, argv); @@ -53,7 +42,7 @@ int main(int argc, char** argv) { // Loop through all available benchmarks sensors::runSensorsAndReduceOutput(processor_name, "pre"); - auto output = benchmarks::runAllBenchmarks(M, N1, N2, N3, K, iters + 1); // add one iteration since we'll drop the first one + auto output = benchmarks::runAllBenchmarks(sizes, iters + 1); // add one iteration since we'll drop the first one sensors::runSensorsAndReduceOutput(processor_name, "post"); benchmarks::printBenchmarkOutput(output, iters); From 1cbb2a5b2b5a23f6ee8d09fd8d4463b343119d6b Mon Sep 17 00:00:00 2001 From: Caleb Schilly Date: Thu, 12 Jun 2025 14:21:18 -0400 Subject: [PATCH 05/10] #10: fix handling of defaults --- src/slow_node.cc | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/slow_node.cc b/src/slow_node.cc index 1b48ad6..dd908ce 100644 --- a/src/slow_node.cc +++ b/src/slow_node.cc @@ -24,15 +24,23 @@ int main(int argc, char** argv) { if (argc > 1) { iters = atoi(argv[1]); - sizes.push_back(atoi(argv[2])); /* N1 */ - sizes.push_back(atoi(argv[3])); /* M2 */ - sizes.push_back(atoi(argv[4])); /* N2 */ - sizes.push_back(atoi(argv[5])); /* M3 */ - sizes.push_back(atoi(argv[6])); /* N3 */ - sizes.push_back(atoi(argv[7])); /* K3 */ - sizes.push_back(atoi(argv[8])); /* N4 */ + N1 = atoi(argv[2]); + M2 = atoi(argv[3]); + N2 = atoi(argv[4]); + M3 = atoi(argv[5]); + N3 = atoi(argv[6]); + K3 = atoi(argv[7]); + N4 = atoi(argv[8]); } + sizes.push_back(N1); + sizes.push_back(M2); + sizes.push_back(N2); + sizes.push_back(M3); + sizes.push_back(N3); + sizes.push_back(K3); + sizes.push_back(N4); + MPI_Init(&argc, &argv); Kokkos::initialize(argc, argv); From 590af548fbfd54fb825f99d3dd0005f986b40dde Mon Sep 17 00:00:00 2001 From: Nicole Lemaster Slattengren Date: Wed, 13 Aug 2025 11:08:20 -0600 Subject: [PATCH 06/10] #10: fix bug in slow iteration printing --- detection/core/SlowNodeDetector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection/core/SlowNodeDetector.py b/detection/core/SlowNodeDetector.py index bb69fcc..706a17d 100644 --- a/detection/core/SlowNodeDetector.py +++ b/detection/core/SlowNodeDetector.py @@ -287,7 +287,7 @@ def detect(self, print_results=True): slowest_iteration = np.argmax(breakdown) rank_with_slowest_iteration = r_id if len(all_ranks_slowest_iters) > 0: - all_ranks_slowest_iters = dict(sorted(all_ranks_slowest_iters.items(), reverse=True, key=lambda item: item[1])) + all_ranks_slowest_iters = dict(sorted(all_ranks_slowest_iters.items(), reverse=True, key=lambda item: item[1][1])) # Print results if print_results: From c24dc67d6d66a8c4933c9f7a4b12f04faaebefd6 Mon Sep 17 00:00:00 2001 From: Nicole Lemaster Slattengren Date: Wed, 13 Aug 2025 11:12:48 -0600 Subject: [PATCH 07/10] #10: Sort nodes by worst rank --- detection/core/SlowNodeDetector.py | 49 +++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/detection/core/SlowNodeDetector.py b/detection/core/SlowNodeDetector.py index 706a17d..060ca99 100644 --- a/detection/core/SlowNodeDetector.py +++ b/detection/core/SlowNodeDetector.py @@ -142,6 +142,52 @@ def __sortNodesByExecutionTime(self, nodes: list): # return sorted(nodes, key=lambda n: self.__getNumberOfSlowRanksOnNode(n)) return sorted(node_times, key=lambda t: node_times[t]) + def __sortNodesByMaxRankExecutionTime(self, nodes: list): + """ + Takes in a list of node names and sorts them based on total execution time. + The fastest nodes will be first, and the slowest will be last. + """ + node_times = {} + for r, n in self.__rank_to_node_map.items(): + if n in nodes: + if n not in node_times: + node_times[n] = 0.0 + if self.__rank_times[r] > node_times[n]: + node_times[n] = self.__rank_times[r] + # Alternative: + # return sorted(nodes, key=lambda n: self.__getNumberOfSlowRanksOnNode(n)) + return sorted(node_times, key=lambda t: node_times[t]) + + def __sortNodesByNodeDevFromAvgExecutionTime(self, nodes: list): + """ + Takes in a list of node names and sorts them based on how much they deviate + from the average total execution time. + """ + node_times = {} + for r, n in self.__rank_to_node_map.items(): + if n in nodes: + if n not in node_times: + node_times[n] = 0.0 + node_times[n] += self.__rank_times[r] + avg = np.mean(list(node_times.values())) + return sorted(node_times, key=lambda t: abs(node_times[t]-avg)) + + def __sortNodesByRankDevFromAvgExecutionTime(self, nodes: list): + """ + Takes in a list of node names and sorts them based on how much they deviate + from the average total execution time. + """ + avg = np.mean(list(self.__rank_times.values())) + node_dev_times = {} + for r, n in self.__rank_to_node_map.items(): + if n in nodes: + if n not in node_dev_times: + node_dev_times[n] = 0.0 + this_dev_time = abs(self.__rank_times[r]-avg) + if this_dev_time > node_dev_times[n]: + node_dev_times[n] = this_dev_time + return sorted(node_dev_times, key=lambda t: node_dev_times[t]) + def __findHighOutliers(self, data): """ Finds data points that are some percentage (given by self.__threshold_pct) @@ -386,7 +432,8 @@ def createHostfile(self): elif num_good_nodes > self.__num_nodes: n_nodes_to_drop = num_good_nodes - self.__num_nodes assert n_nodes_to_drop > 0, f"Cannot drop {n_nodes_to_drop}" - sorted_nodes = self.__sortNodesByExecutionTime(good_node_names) + #sorted_nodes = self.__sortNodesByExecutionTime(good_node_names) + sorted_nodes = self.__sortNodesByMaxRankExecutionTime(good_node_names) print( f"Since the SlowNodeDetector originally found {num_good_nodes} good node{s}, " f"but only {self.__num_nodes} are needed, the following nodes will also be " From 0ecf3e9e29bab035421f95ab3d06e6cb96aa3d1f Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 4 Sep 2025 14:46:11 -0700 Subject: [PATCH 08/10] #10: revise comments to be more accurate --- detection/core/SlowNodeDetector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/detection/core/SlowNodeDetector.py b/detection/core/SlowNodeDetector.py index 060ca99..2395f07 100644 --- a/detection/core/SlowNodeDetector.py +++ b/detection/core/SlowNodeDetector.py @@ -144,8 +144,9 @@ def __sortNodesByExecutionTime(self, nodes: list): def __sortNodesByMaxRankExecutionTime(self, nodes: list): """ - Takes in a list of node names and sorts them based on total execution time. - The fastest nodes will be first, and the slowest will be last. + Takes in a list of node names and sorts them based on maximum rank + execution time on the node. The fastest nodes will be first, and the + slowest will be last. """ node_times = {} for r, n in self.__rank_to_node_map.items(): @@ -174,8 +175,9 @@ def __sortNodesByNodeDevFromAvgExecutionTime(self, nodes: list): def __sortNodesByRankDevFromAvgExecutionTime(self, nodes: list): """ - Takes in a list of node names and sorts them based on how much they deviate - from the average total execution time. + Takes in a list of node names and sorts them based on the maximum + rank deviation from the rank-avg execution time. + """ avg = np.mean(list(self.__rank_times.values())) node_dev_times = {} From 0eaafd02162c7136d9b66e702b8646b09da1cb5d Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 4 Sep 2025 14:46:36 -0700 Subject: [PATCH 09/10] #10: fill random output buffers so pages are not allocated during runtime --- src/benchmarks.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/benchmarks.cc b/src/benchmarks.cc index 116d35b..dd49412 100644 --- a/src/benchmarks.cc +++ b/src/benchmarks.cc @@ -79,7 +79,7 @@ benchmark_results_t runBenchmarkLevel2(int M, int N, int iters) { Kokkos::Random_XorShift64_Pool pool(123); Kokkos::fill_random(x, pool, 10.0); Kokkos::fill_random(y, pool, 10.0); - // Kokkos::fill_random(A, pool, 10.0); + Kokkos::fill_random(A, pool, 10.0); std::vector iter_timings; double total_time = 0.0; @@ -119,6 +119,7 @@ benchmark_results_t runBenchmarkLevel3(int M, int N, int K, int iters) { Kokkos::Random_XorShift64_Pool pool(123); Kokkos::fill_random(A, pool, 10.0); Kokkos::fill_random(B, pool, 10.0); + Kokkos::fill_random(C, pool, 10.0); std::vector iter_timings; From 6a4c91711e2f92ef10f578c3081177ed2bf8abc6 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 4 Sep 2025 14:52:25 -0700 Subject: [PATCH 10/10] #10: add barriers after each benchmark runs --- src/benchmarks.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/benchmarks.cc b/src/benchmarks.cc index dd49412..faedea5 100644 --- a/src/benchmarks.cc +++ b/src/benchmarks.cc @@ -219,7 +219,9 @@ all_results_t runAllBenchmarks(std::vector sizes, int iters) { auto b = static_cast(i); std::string benchmark_str = benchmarkToString(b); all_results[benchmark_str + "_double"] = runBenchmark(b, sizes, iters); + MPI_Barrier(MPI_COMM_WORLD); all_results[benchmark_str + "_complex"] = runBenchmark>(b, sizes, iters); + MPI_Barrier(MPI_COMM_WORLD); } return all_results; }