diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a0a964..3ac83db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ set(SLOW_NODE_EXE slow_node) add_executable( ${SLOW_NODE_EXE} "${CMAKE_SOURCE_DIR}/src/slow_node.cc" + "${CMAKE_SOURCE_DIR}/src/benchmarks.cc" "${CMAKE_SOURCE_DIR}/src/sensors.cc" "${CMAKE_SOURCE_DIR}/src/freq.cc" ) diff --git a/detection/core/SlowNodeDetector.py b/detection/core/SlowNodeDetector.py index 43b8328..2395f07 100644 --- a/detection/core/SlowNodeDetector.py +++ b/detection/core/SlowNodeDetector.py @@ -35,7 +35,7 @@ class SlowNodeDetector: """ def __init__( - self, path, sensors, num_nodes, pct, spn, rpn, plot_rank_breakdowns): + self, path, sensors, num_nodes, pct, benchmark, type, spn, rpn, plot_rank_breakdowns): # Create empty dicts for storing data self.__rank_times = {} self.__rank_breakdowns = {} @@ -49,6 +49,8 @@ def __init__( self.__sensors_output_file = sensors self.__num_nodes = int(num_nodes) if num_nodes is not None else None self.__threshold_pct = float(pct) + self.__benchmark = benchmark + self.__datatype = type self.__spn = int(spn) self.__rpn = int(rpn) self.__rps = self.__rpn / self.__spn @@ -87,7 +89,7 @@ def __parseOutput(self): """Parses text output from slow_node.cc""" self.__rank_times, \ self.__rank_breakdowns, \ - self.__rank_to_node_map = parseOutput(self.__filepath) + self.__rank_to_node_map = parseOutput(self.__filepath, self.__benchmark, self.__datatype) self.__num_ranks = len(self.__rank_times) @@ -140,6 +142,54 @@ def __sortNodesByExecutionTime(self, nodes: list): # return sorted(nodes, key=lambda n: self.__getNumberOfSlowRanksOnNode(n)) return sorted(node_times, key=lambda t: node_times[t]) + def __sortNodesByMaxRankExecutionTime(self, nodes: list): + """ + Takes in a list of node names and sorts them based on maximum rank + execution time on the node. The fastest nodes will be first, and the + slowest will be last. + """ + node_times = {} + for r, n in self.__rank_to_node_map.items(): + if n in nodes: + if n not in node_times: + node_times[n] = 0.0 + if self.__rank_times[r] > node_times[n]: + node_times[n] = self.__rank_times[r] + # Alternative: + # return sorted(nodes, key=lambda n: self.__getNumberOfSlowRanksOnNode(n)) + return sorted(node_times, key=lambda t: node_times[t]) + + def __sortNodesByNodeDevFromAvgExecutionTime(self, nodes: list): + """ + Takes in a list of node names and sorts them based on how much they deviate + from the average total execution time. + """ + node_times = {} + for r, n in self.__rank_to_node_map.items(): + if n in nodes: + if n not in node_times: + node_times[n] = 0.0 + node_times[n] += self.__rank_times[r] + avg = np.mean(list(node_times.values())) + return sorted(node_times, key=lambda t: abs(node_times[t]-avg)) + + def __sortNodesByRankDevFromAvgExecutionTime(self, nodes: list): + """ + Takes in a list of node names and sorts them based on the maximum + rank deviation from the rank-avg execution time. + + """ + avg = np.mean(list(self.__rank_times.values())) + node_dev_times = {} + for r, n in self.__rank_to_node_map.items(): + if n in nodes: + if n not in node_dev_times: + node_dev_times[n] = 0.0 + this_dev_time = abs(self.__rank_times[r]-avg) + if this_dev_time > node_dev_times[n]: + node_dev_times[n] = this_dev_time + return sorted(node_dev_times, key=lambda t: node_dev_times[t]) + def __findHighOutliers(self, data): """ Finds data points that are some percentage (given by self.__threshold_pct) @@ -285,12 +335,13 @@ def detect(self, print_results=True): slowest_iteration = np.argmax(breakdown) rank_with_slowest_iteration = r_id if len(all_ranks_slowest_iters) > 0: - all_ranks_slowest_iters = dict(sorted(all_ranks_slowest_iters.items(), reverse=True, key=lambda item: item[1])) + all_ranks_slowest_iters = dict(sorted(all_ranks_slowest_iters.items(), reverse=True, key=lambda item: item[1][1])) # Print results if print_results: s = self.__s(slow_rank_ids) n = len(str(abs(int(self.__num_ranks)))) + print(f"\nPrinting analysis from {self.__benchmark}_{self.__datatype} benchmark...") print("\n----------------------------------------------------------") print("Across-Rank Analysis") print() @@ -383,7 +434,8 @@ def createHostfile(self): elif num_good_nodes > self.__num_nodes: n_nodes_to_drop = num_good_nodes - self.__num_nodes assert n_nodes_to_drop > 0, f"Cannot drop {n_nodes_to_drop}" - sorted_nodes = self.__sortNodesByExecutionTime(good_node_names) + #sorted_nodes = self.__sortNodesByExecutionTime(good_node_names) + sorted_nodes = self.__sortNodesByMaxRankExecutionTime(good_node_names) print( f"Since the SlowNodeDetector originally found {num_good_nodes} good node{s}, " f"but only {self.__num_nodes} are needed, the following nodes will also be " diff --git a/detection/detect_slow_nodes.py b/detection/detect_slow_nodes.py index cdc4bef..510451a 100644 --- a/detection/detect_slow_nodes.py +++ b/detection/detect_slow_nodes.py @@ -17,6 +17,8 @@ def main(): parser.add_argument('-s', '--sensors', help='Absolute or relative path to the sensors file that will be analyzed', default=None) parser.add_argument('-N', '--num_nodes', help='The number of nodes required by the application', default=None) parser.add_argument('-t', '--threshold', help='Percentage above average time that indicates a "slow" rank', default=0.05) + parser.add_argument('-b', '--benchmark', help='Benchmark to analyze: [level1, level2, level3, dpotrf]', default='level3') + parser.add_argument('-d', '--datatype', help='Datatype of benchmark to analyze: [double, complex]', default='double') parser.add_argument('-spn', '--spn', help='Number of sockets per node', default=2) parser.add_argument('-rpn', '--rpn', help='Number of ranks per node', default=48) parser.add_argument('-p', '--plot_all_ranks', action='store_true', help='Plot the breakdowns for every rank') @@ -30,6 +32,8 @@ def main(): sensors=sensors_filepath, num_nodes=args.num_nodes, pct=args.threshold, + benchmark=args.benchmark, + type=args.datatype, spn=args.spn, rpn=args.rpn, plot_rank_breakdowns=args.plot_all_ranks) diff --git a/detection/utils/Parse.py b/detection/utils/Parse.py index 4054a93..3101033 100644 --- a/detection/utils/Parse.py +++ b/detection/utils/Parse.py @@ -7,38 +7,46 @@ def matchRegex(pattern: str, line: str): return tuple(match.groups()) raise RuntimeError(f"regex matching failed on line {line}") -def parseOutput(slownode_file): +def parseOutput(slownode_file, benchmark, datatype): """Parses text output from slow_node.cc""" rank_times = {} rank_breakdowns = {} rank_to_node_map = {} + is_parsing=False with open(slownode_file, "r") as output: for line in output: - if line.startswith("gather"): - # splits: ['gather', rank_info, total_time, 'breakdown', [times]] - splits = line.split(":") - - # 1. Determine the Rank ID (and node name, if present) - raw_rank_info = splits[1].strip() - # raw_rank_info = 'rank_id (node)' - rank_info = re.findall( - r"(\d+)\s+\(([^)]+)\)", - raw_rank_info - )[0] - rank_id = int(rank_info[0]) - node_name = rank_info[1] - rank_to_node_map[rank_id] = node_name - - # 2. Get the total time for the current rank - total_time = float(splits[2].strip()) - - # 3. Isolate the times for each iteration on the current rank - breakdown = splits[4].strip() - breakdown_list = [float(t) for t in breakdown.split(" ")] - - # Populate rank data dicts - rank_times[rank_id] = total_time - rank_breakdowns[rank_id] = breakdown_list + if line.startswith(f"{benchmark}_{datatype}"): + is_parsing = True + + if is_parsing: + if line.startswith("gather"): + # splits: ['gather', rank_info, total_time, 'breakdown', [times]] + splits = line.split(":") + + # 1. Determine the Rank ID (and node name, if present) + raw_rank_info = splits[1].strip() + # raw_rank_info = 'rank_id (node)' + rank_info = re.findall( + r"(\d+)\s+\(([^)]+)\)", + raw_rank_info + )[0] + rank_id = int(rank_info[0]) + node_name = rank_info[1] + rank_to_node_map[rank_id] = node_name + + # 2. Get the total time for the current rank + total_time = float(splits[2].strip()) + + # 3. Isolate the times for each iteration on the current rank + breakdown = splits[4].strip() + breakdown_list = [float(t) for t in breakdown.split(" ")] + + # Populate rank data dicts + rank_times[rank_id] = total_time + rank_breakdowns[rank_id] = breakdown_list + + elif line.strip() == "": + is_parsing = False return rank_times, rank_breakdowns, rank_to_node_map @@ -52,7 +60,7 @@ def parseSensors(sensors_file): with open(sensors_file, 'r') as sensor_data: for line in sensor_data: if line.startswith("Node"): - pattern = r"Node (\w+), Socket (\d+), Core (\d+): (\d+)(?:°C| C), (\d+) KHz" + pattern = r"Node (\w+), Socket (\d+), Core (\d+): (\d+)(?:°C| C), (?:-|)(\d+) KHz" node_name, \ socket_str, \ diff --git a/src/benchmarks.cc b/src/benchmarks.cc new file mode 100644 index 0000000..faedea5 --- /dev/null +++ b/src/benchmarks.cc @@ -0,0 +1,292 @@ +#include +#include +#include + +#include "ops.h" +#include "benchmarks.h" + +#include + +namespace benchmarks { + +template <> +std::string typeToString() { + return "double "; +} + +template <> +std::string typeToString>() { + return "complex"; +} + +std::string benchmarkToString(const benchmark_types& b) { + switch (b) { + case level1: return "level1"; + case level2: return "level2"; + case level3: return "level3"; + case dpotrf: return "dpotrf"; + default: return "unknown"; + } +} + +template +benchmark_results_t runBenchmarkLevel1(int N, int iters) { + Kokkos::View x("x", N); + Kokkos::View y("y", N); + + T* x_ptr = x.data(); + T* y_ptr = y.data(); + + Kokkos::Random_XorShift64_Pool pool(123); + Kokkos::fill_random(x, pool, 10.0); + Kokkos::fill_random(y, pool, 10.0); + + std::vector iter_timings; + double total_time = 0.0; + + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < iters; i++) { + Kokkos::Timer timer; + ops::level1(N, x_ptr, y_ptr); + Kokkos::fence(); + + // Skip the first iteration + if (i > 0) { + double time = timer.seconds(); + total_time += time; + iter_timings.push_back(time); + } + } + + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + std::cout << "[level1 " << typeToString() << "] (N=" << N << ") rank: " << rank << ", total_time=" << total_time << std::endl; + + return std::make_tuple(iter_timings, total_time); +} + +template +benchmark_results_t runBenchmarkLevel2(int M, int N, int iters) { + Kokkos::View x("x", M); + Kokkos::View y("y", N); + Kokkos::View A("A", M, N); + + T* x_ptr = x.data(); + T* y_ptr = y.data(); + T* A_ptr = A.data(); + + Kokkos::Random_XorShift64_Pool pool(123); + Kokkos::fill_random(x, pool, 10.0); + Kokkos::fill_random(y, pool, 10.0); + Kokkos::fill_random(A, pool, 10.0); + + std::vector iter_timings; + double total_time = 0.0; + + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < iters; i++) { + Kokkos::Timer timer; + ops::level2(M, N, T(1.0), x_ptr, y_ptr, A_ptr, N); + Kokkos::fence(); + + // Skip the first iteration + if (i > 0) { + double time = timer.seconds(); + total_time += time; + iter_timings.push_back(time); + } + } + + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + std::cout << "[level2 " << typeToString() << "] (M=" << M << ", N=" << N << ") rank: " << rank << ", total_time=" << total_time << std::endl; + + return std::make_tuple(iter_timings, total_time); +} + +template +benchmark_results_t runBenchmarkLevel3(int M, int N, int K, int iters) { + Kokkos::View A("A", M, N); + Kokkos::View B("B", N, K); + Kokkos::View C("C", M, K); + + T* A_ptr = A.data(); + T* B_ptr = B.data(); + T* C_ptr = C.data(); + + Kokkos::Random_XorShift64_Pool pool(123); + Kokkos::fill_random(A, pool, 10.0); + Kokkos::fill_random(B, pool, 10.0); + Kokkos::fill_random(C, pool, 10.0); + + std::vector iter_timings; + + double total_time = 0.0; + + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < iters; i++) { + Kokkos::Timer timer; + ops::level3(M, K, N, T(1.0), A_ptr, N, B_ptr, K, T(0.0), C_ptr, K); + Kokkos::fence(); + + // Skip the first iteration + if (i > 0) { + double time = timer.seconds(); + total_time += time; + iter_timings.push_back(time); + } + } + + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + std::cout << "[level3 " << typeToString() << "] (M=" << M << ", N=" << N << ", K=" << K << ") rank: " << rank << ", total_time=" << total_time << std::endl; + + return std::make_tuple(iter_timings, total_time); +} + +template +benchmark_results_t runBenchmarkDPOTRF(int N, int iters) { + Kokkos::View A("A", N, N); + + Kokkos::Random_XorShift64_Pool pool(123); + Kokkos::fill_random(A, pool, 10.0); + + // Make A symmetric positive definite + Kokkos::parallel_for("MakeSPD", N, KOKKOS_LAMBDA(int i) { + for (int j = 0; j < N; j++) { + A(i, j) = A(i, j) + A(j, i); + } + }); + + Kokkos::fence(); + + T* A_ptr = A.data(); + + std::vector iter_timings; + double total_time = 0.0; + + long long N_ll = static_cast(N); + char uplo = 'L'; + + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < iters; i++) { + long long info; + Kokkos::Timer timer; + ops::dpotrf(uplo, N_ll, A_ptr, N_ll, &info); + + Kokkos::fence(); + + // Skip the first iteration + if (i > 0) { + double time = timer.seconds(); + total_time += time; + iter_timings.push_back(time); + } + } + + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + std::cout << "[dpotrf " << typeToString() << "] (N=" << N << ") rank: " << rank << ", total_time=" << total_time << std::endl; + + return std::make_tuple(iter_timings, total_time); +} + +template +benchmark_results_t runBenchmark(benchmark_types b, std::vector sizes, int iters) { + switch (b) { + case level1: + return runBenchmarkLevel1(sizes[0], iters); + case level2: + return runBenchmarkLevel2(sizes[1], sizes[2], iters); + case level3: + return runBenchmarkLevel3(sizes[3], sizes[4], sizes[5], iters); + case dpotrf: + return runBenchmarkDPOTRF(sizes[6], iters); + default: + throw std::invalid_argument("Unsupported benchmark type"); + } +} + +all_results_t runAllBenchmarks(std::vector sizes, int iters) { + all_results_t all_results; + for (int i=0; i < benchmark_types::num_benchmarks; i++) { + auto b = static_cast(i); + std::string benchmark_str = benchmarkToString(b); + all_results[benchmark_str + "_double"] = runBenchmark(b, sizes, iters); + MPI_Barrier(MPI_COMM_WORLD); + all_results[benchmark_str + "_complex"] = runBenchmark>(b, sizes, iters); + MPI_Barrier(MPI_COMM_WORLD); + } + return all_results; +} + +void printBenchmarkOutput(all_results_t benchmark_results, int iters) +{ + int rank = -1; + int num_ranks = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &num_ranks); + + if (rank == 0) { + std::cout << "num_ranks: " << num_ranks << std::endl; + } + + for (const auto& [benchmark_str, benchmark_results]: benchmark_results) { + char processor_name[MPI_MAX_PROCESSOR_NAME]; + int name_len; + MPI_Get_processor_name(processor_name, &name_len); + + auto const& [iter_timings, total_time] = benchmark_results; + + std::vector all_times; + all_times.resize(num_ranks); + + std::vector all_iter_times; + all_iter_times.resize(num_ranks * iters); + + std::vector all_processor_names; + all_processor_names.resize(num_ranks * MPI_MAX_PROCESSOR_NAME); + + MPI_Gather( + &total_time, 1, MPI_DOUBLE, + &all_times[0], 1, MPI_DOUBLE, 0, + MPI_COMM_WORLD + ); + + MPI_Gather( + &iter_timings[0], iters, MPI_DOUBLE, + &all_iter_times[0], iters, MPI_DOUBLE, 0, + MPI_COMM_WORLD + ); + + MPI_Gather( + &processor_name, MPI_MAX_PROCESSOR_NAME, MPI_CHAR, + &all_processor_names[0], MPI_MAX_PROCESSOR_NAME, MPI_CHAR, 0, + MPI_COMM_WORLD + ); + + if (rank == 0) { + int cur_rank = 0; + int cur = 0; + std::cout << std::endl << benchmark_str << std::endl; + for (auto&& time : all_times) { + std::cout << "gather: " << cur_rank << " (" + << std::string(&all_processor_names[cur_rank * MPI_MAX_PROCESSOR_NAME]) + << "): " << time << ": breakdown: "; + for (int i = cur; i < iters + cur; i++) { + std::cout << all_iter_times[i] << " "; + } + std::cout << std::endl; + cur += iters; + cur_rank++; + } + } + } +} +} // end namespace benchmarks diff --git a/src/benchmarks.h b/src/benchmarks.h new file mode 100644 index 0000000..eeb004f --- /dev/null +++ b/src/benchmarks.h @@ -0,0 +1,46 @@ +#ifndef SRC_BENCHMARKS_H +#define SRC_BENCHMARKS_H + +#include +#include + +namespace benchmarks { + +using benchmark_results_t = std::tuple, double>; +using all_results_t = std::unordered_map; + +enum benchmark_types { + level1, + level2, + level3, + dpotrf, + num_benchmarks +}; + +template +std::string typeToString(); + +std::string benchmarkToString(const benchmark_types& b); + +template +benchmark_results_t runBenchmarkLevel1(int N, int iters); + +template +benchmark_results_t runBenchmarkLevel2(int M, int N, int iters); + +template +benchmark_results_t runBenchmarkLevel3(int M, int N, int K, int iters); + +template +benchmark_results_t runBenchmarkDPOTRF(int N, int iters); + +template +benchmark_results_t runBenchmark(benchmark_types b, std::vector sizes, int iters); + +all_results_t runAllBenchmarks(std::vector sizes, int iters); + +void printBenchmarkOutput(all_results_t benchmark_results, int iters); + +} // end namespace benchmarks + +#endif // SRC_BENCHMARKS_H diff --git a/src/ops.h b/src/ops.h new file mode 100644 index 0000000..4327f0b --- /dev/null +++ b/src/ops.h @@ -0,0 +1,32 @@ +#ifndef SRC_OPS_H +#define SRC_OPS_H + +namespace ops { + +template +constexpr bool isDouble(); + +template +constexpr bool isComplex(); + +template +void level1(int N, const T* x, const T* y); + +template +void level2(int M, int N, T alpha, const T* x, const T* y, T* A, int lda); + +template +void level3( + int M, int N, int K, + T alpha, const T* A, int lda, + const T* B, int ldb, + T beta, T* C, int ldc); + +template +void dpotrf(char uplo, long long n, T* a, long long lda, long long* info); + +} // end namespace ops + +#include "ops_impl.h" + +#endif // SRC_OPS_H diff --git a/src/ops_impl.h b/src/ops_impl.h new file mode 100644 index 0000000..5a051df --- /dev/null +++ b/src/ops_impl.h @@ -0,0 +1,139 @@ +#ifndef SRC_OPS_IMPL_H +#define SRC_OPS_IMPL_H + +#include +#include + +#include + +#include "ops.h" + +namespace ops { + +/////////////////////////////////////////////////////////////////////////////// +// Helpers + +template +constexpr bool isDouble() { + if constexpr (std::is_same_v) { + return true; + } + return false; +} + +template +constexpr bool isComplex() { + if constexpr (std::is_same_v>) { + return true; + } + return false; +} + +static void warnWrongType(const std::string& benchmark) { + std::cout << "Warning: Type must be either double or Kokkos::complex. Skipping " << benchmark << " benchmark." << std::endl; +} + +/////////////////////////////////////////////////////////////////////////////// +// LEVEL 1 + +template +void level1(int N, const T* x, const T* y) { + if constexpr (isDouble()) { + cblas_ddot(N, x, 1, y, 1); + } else if constexpr (isComplex) { + T result; + cblas_zdotu_sub(N, x, 1, y, 1, &result); + } else { + warnWrongType("level1"); + } +} + +/////////////////////////////////////////////////////////////////////////////// +// LEVEL 2 + +template +void level2( + int M, + int N, + T alpha, + const T* x, + const T* y, + T* A, + int lda) +{ + if constexpr (isDouble()) { + cblas_dger(CblasRowMajor, M, N, alpha, x, 1, y, 1, A, lda); + } else if constexpr (isComplex()) { + cblas_zgeru( + CblasRowMajor, + M, N, + reinterpret_cast(&alpha), + reinterpret_cast(x), 1, + reinterpret_cast(y), 1, + reinterpret_cast(A), lda + ); + } else { + warnWrongType("level2"); + } +} + +/////////////////////////////////////////////////////////////////////////////// +// LEVEL 3 + +template +void level3( + int M, + int N, + int K, + T alpha, + const T* A, + int lda, + const T* B, + int ldb, + T beta, + T* C, + int ldc) +{ + if constexpr (isDouble()) { + cblas_dgemm( + CblasRowMajor, CblasNoTrans, CblasNoTrans, + M, N, K, + alpha, + A, lda, + B, ldb, + beta, + C, ldc + ); + } else if constexpr (isComplex()) { + cblas_zgemm( + CblasRowMajor, CblasNoTrans, CblasNoTrans, + M, N, K, + reinterpret_cast(&alpha), + reinterpret_cast(A), lda, + reinterpret_cast(B), ldb, + reinterpret_cast(&beta), + reinterpret_cast(C), ldc); + } else { + warnWrongType("level3"); + } + + +} + +/////////////////////////////////////////////////////////////////////////////// +// DPOTRF + +template +void dpotrf(char uplo, long long n, T* a, long long lda, long long* info) { + if constexpr (isDouble()) { + dpotrf_(&uplo, &n, a, &lda, info); + } else if constexpr (isComplex()) { + zpotrf_(&uplo, &n, reinterpret_cast(a), &lda, info); + } else { + warnWrongType("dpotrf"); + } +} + +} // end namespace ops + +#endif // SRC_OPS_IMPL_H diff --git a/src/sensors.cc b/src/sensors.cc index a45501c..a3d9d11 100644 --- a/src/sensors.cc +++ b/src/sensors.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include #include diff --git a/src/slow_node.cc b/src/slow_node.cc index f312aa2..dd908ce 100644 --- a/src/slow_node.cc +++ b/src/slow_node.cc @@ -1,135 +1,58 @@ #include "sensors.h" +#include "benchmarks.h" -#include #include #include static int iters = 100; -static int M = 128; -static int N = 128; -static int K = 128; - -std::tuple, double> runBenchmark() { - Kokkos::View A("A", M, N); - Kokkos::View B("B", N, K); - Kokkos::View C("C", M, K); - - double* A_ptr = A.data(); - double* B_ptr = B.data(); - double* C_ptr = C.data(); - - Kokkos::Random_XorShift64_Pool pool(123); - Kokkos::fill_random(A, pool, 10.0); - Kokkos::fill_random(B, pool, 10.0); - - std::vector iter_timings; - - double total_time = 0.0; - - MPI_Barrier(MPI_COMM_WORLD); - - for (int i = 0; i < iters; i++) { - Kokkos::Timer timer; - cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, - M, // number of rows in C (and A) - K, // number of columns in C (and B) - N, // shared inner dimension (columns of A, rows of B) - 1.0, // alpha - A_ptr, // matrix A pointer - N, // leading dimension of A (because A is M×N) - B_ptr, // matrix B pointer - K, // leading dimension of B (because B is N×K) - 0.0, // beta - C_ptr, // matrix C pointer - K); // leading dimension of C (because C is M×K) - Kokkos::fence(); - // Do not count the first iteration - if (i > 0) { - double time = timer.seconds(); - total_time += time; - iter_timings.push_back(time); - } - } - - int rank = -1; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - std::cout << "rank: " << rank << ", total_time=" << total_time << std::endl; +static int N1 = 128; +static int M2 = 128; +static int N2 = 128; +static int M3 = 128; +static int N3 = 128; +static int K3 = 128; +static int N4 = 128; + +/* + * USAGE: ./slow_node + */ +int main(int argc, char** argv) { - return std::make_tuple(iter_timings, total_time); -} + std::vector sizes; -int main(int argc, char** argv) { if (argc > 1) { - iters = atoi(argv[1]) + 1; // add one iteration since we will drop the first one - M = N = K = atoi(argv[2]); + iters = atoi(argv[1]); + N1 = atoi(argv[2]); + M2 = atoi(argv[3]); + N2 = atoi(argv[4]); + M3 = atoi(argv[5]); + N3 = atoi(argv[6]); + K3 = atoi(argv[7]); + N4 = atoi(argv[8]); } - std::cout << "iters: " << iters << ", M=N=K=" << M << std::endl; + + sizes.push_back(N1); + sizes.push_back(M2); + sizes.push_back(N2); + sizes.push_back(M3); + sizes.push_back(N3); + sizes.push_back(K3); + sizes.push_back(N4); MPI_Init(&argc, &argv); Kokkos::initialize(argc, argv); - int rank = -1; - int num_ranks = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &num_ranks); - char processor_name[MPI_MAX_PROCESSOR_NAME]; int name_len; MPI_Get_processor_name(processor_name, &name_len); + // Loop through all available benchmarks sensors::runSensorsAndReduceOutput(processor_name, "pre"); - auto const& [iter_timings, total_time] = runBenchmark(); + auto output = benchmarks::runAllBenchmarks(sizes, iters + 1); // add one iteration since we'll drop the first one sensors::runSensorsAndReduceOutput(processor_name, "post"); - - std::vector all_times; - all_times.resize(num_ranks); - - std::vector all_iter_times; - all_iter_times.resize(num_ranks * iters); - - std::vector all_processor_names; - all_processor_names.resize(num_ranks * MPI_MAX_PROCESSOR_NAME); - - if (rank == 0) { - std::cout << "num_ranks: " << num_ranks << std::endl; - } - - MPI_Gather( - &total_time, 1, MPI_DOUBLE, - &all_times[0], 1, MPI_DOUBLE, 0, - MPI_COMM_WORLD - ); - - MPI_Gather( - &iter_timings[0], iters, MPI_DOUBLE, - &all_iter_times[0], iters, MPI_DOUBLE, 0, - MPI_COMM_WORLD - ); - - MPI_Gather( - &processor_name, MPI_MAX_PROCESSOR_NAME, MPI_CHAR, - &all_processor_names[0], MPI_MAX_PROCESSOR_NAME, MPI_CHAR, 0, - MPI_COMM_WORLD - ); - - if (rank == 0) { - int cur_rank = 0; - int cur = 0; - for (auto&& time : all_times) { - std::cout << "gather: " << cur_rank << " (" - << std::string(&all_processor_names[cur_rank * MPI_MAX_PROCESSOR_NAME]) - << "): " << time << ": breakdown: "; - for (int i = cur; i < iters + cur; i++) { - std::cout << all_iter_times[i] << " "; - } - std::cout << std::endl; - cur += iters; - cur_rank++; - } - } + benchmarks::printBenchmarkOutput(output, iters); Kokkos::finalize(); MPI_Finalize(); diff --git a/tests/logs/slownode.log b/tests/logs/slownode.log index 66678b3..a11dcd4 100644 --- a/tests/logs/slownode.log +++ b/tests/logs/slownode.log @@ -2,37 +2,86 @@ iters: 100, M=N=K=128 iters: 100, M=N=K=128 iters: 100, M=N=K=128 iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -iters: 100, M=N=K=128 -rank: 0, total_time=1.00737 -rank: 3, total_time=1.1036 -rank: 8, total_time=1.20108 -rank: 5, total_time=1.20714 -rank: 9, total_time=6.61778 -rank: 7, total_time=6.6205 -rank: 10, total_time=6.67709 -rank: 11, total_time=6.73952 -rank: 4, total_time=6.74128 -rank: 2, total_time=6.84974 -rank: 6, total_time=6.89012 -rank: 1, total_time=6.95131 -Wrote sensor data for ozark to "sensors_output/sensors_ozark.log" -num_ranks: 12 -gather: 0 (ozark): 1.00737: breakdown: 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 0.00946113 -gather: 1 (ozark): 6.95131: breakdown: 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 0.0688826 -gather: 2 (ozark): 6.84974: breakdown: 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 0.0641464 -gather: 3 (ozark): 1.1036: breakdown: 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 0.00964282 -gather: 4 (ozark): 6.74128: breakdown: 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 0.0650504 -gather: 5 (ozark): 1.20714: breakdown: 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 0.0095877 -gather: 6 (ozark): 6.89012: breakdown: 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 0.186559 -gather: 7 (ozark): 6.6205: breakdown: 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 0.0644537 -gather: 8 (ozark): 1.20108: breakdown: 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 0.00993268 -gather: 9 (ozark): 6.61778: breakdown: 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 0.0642959 -gather: 10 (ozark): 6.67709: breakdown: 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 0.0632457 -gather: 11 (ozark): 6.73952: breakdown: 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 0.06531 +Wrote sensor data to "sensors_pre.log" +[level1 double] rank: 2, total_time=2.35e-05 +[level1 double] rank: 1, total_time=2.3465e-05 +[level1 double] rank: 3, total_time=2.3176e-05 +[level1 double] rank: 0, total_time=2.6729e-05 +[level1 complex] rank: 0, total_time=1.8592e-05 +[level1 complex] rank: 3, total_time=1.8519e-05 +[level1 complex] rank: 2, total_time=1.9089e-05 +[level1 complex] rank: 1, total_time=2.1363e-05 +[level2 double] rank: 3, total_time=0.000377028 +[level2 double] rank: 2, total_time=0.000375657 +[level2 double] rank: 0, total_time=0.000378346 +[level2 double] rank: 1, total_time=0.000377205 +[level2 complex] rank: 1, total_time=0.000860855 +[level2 complex] rank: 3, total_time=0.000864736 +[level2 complex] rank: 2, total_time=0.000864709 +[level2 complex] rank: 0, total_time=0.000867368 +[level3 double] rank: 0, total_time=0.013612 +[level3 double] rank: 1, total_time=0.0136195 +[level3 double] rank: 2, total_time=0.0136247 +[level3 double] rank: 3, total_time=0.0136833 +[level3 complex] rank: 3, total_time=0.0535708 +[level3 complex] rank: 1, total_time=0.0536701 +[level3 complex] rank: 2, total_time=0.0538132 +[level3 complex] rank: 0, total_time=0.0538252 +[dpotrf double] rank: 2, total_time=3.607e-05 +[dpotrf double] rank: 3, total_time=3.5959e-05 +[dpotrf double] rank: 0, total_time=3.568e-05 +[dpotrf double] rank: 1, total_time=3.6751e-05 +[dpotrf complex] rank: 2, total_time=0.0010499 +[dpotrf complex] rank: 1, total_time=0.00105277 +[dpotrf complex] rank: 3, total_time=0.00105296 +[dpotrf complex] rank: 0, total_time=0.00106083 +Wrote sensor data to "sensors_post.log" +num_ranks: 4 + +dpotrf_complex +gather: 0 (ozark): 0.00106083: breakdown: 1.2762e-05 1.1106e-05 1.0822e-05 1.0656e-05 1.0811e-05 1.0742e-05 1.0703e-05 1.069e-05 1.0638e-05 1.0725e-05 1.0536e-05 1.7613e-05 1.2004e-05 1.0648e-05 1.0615e-05 1.0659e-05 1.07e-05 1.0593e-05 1.0581e-05 1.0604e-05 1.0641e-05 1.0629e-05 1.0607e-05 1.0568e-05 1.0478e-05 1.0637e-05 1.0607e-05 1.0511e-05 1.0555e-05 1.0709e-05 1.0533e-05 1.0573e-05 1.0611e-05 1.06e-05 1.0541e-05 1.0567e-05 1.0562e-05 1.0609e-05 1.0652e-05 1.0511e-05 1.0596e-05 1.0561e-05 1.0536e-05 1.0604e-05 1.0589e-05 1.0563e-05 1.0617e-05 1.06e-05 1.055e-05 1.063e-05 1.0494e-05 1.0502e-05 1.0559e-05 1.062e-05 1.0596e-05 1.052e-05 1.0565e-05 1.0513e-05 1.0587e-05 1.0604e-05 1.0517e-05 1.0588e-05 1.0612e-05 1.0583e-05 1.0628e-05 1.0528e-05 1.0574e-05 1.0555e-05 1.0527e-05 1.0609e-05 1.0624e-05 1.0526e-05 1.061e-05 1.0587e-05 1.0579e-05 1.0555e-05 1.0519e-05 1.0593e-05 1.0607e-05 1.0634e-05 1.0571e-05 1.0636e-05 1.0644e-05 1.0575e-05 1.0572e-05 1.059e-05 1.0548e-05 1.063e-05 1.0611e-05 1.0667e-05 1.061e-05 1.0701e-05 1.0654e-05 1.0705e-05 1.0666e-05 1.0587e-05 1.0671e-05 1.0655e-05 1.0671e-05 1.0326e-321 +gather: 1 (ozark): 0.00105277: breakdown: 1.1991e-05 1.089e-05 1.079e-05 1.0664e-05 1.061e-05 1.0528e-05 1.0521e-05 1.0519e-05 1.0559e-05 1.0628e-05 1.0618e-05 1.5703e-05 1.0782e-05 1.0575e-05 1.0636e-05 1.0599e-05 1.0622e-05 1.0719e-05 1.0554e-05 1.0471e-05 1.0515e-05 1.0628e-05 1.0491e-05 1.0667e-05 1.0623e-05 1.0477e-05 1.0521e-05 1.0472e-05 1.0525e-05 1.0514e-05 1.0532e-05 1.0529e-05 1.0596e-05 1.063e-05 1.0552e-05 1.0611e-05 1.0528e-05 1.0601e-05 1.0565e-05 1.0533e-05 1.0489e-05 1.0516e-05 1.055e-05 1.0492e-05 1.0569e-05 1.0574e-05 1.0488e-05 1.0563e-05 1.0611e-05 1.0617e-05 1.0565e-05 1.0599e-05 1.0536e-05 1.0573e-05 1.0533e-05 1.0526e-05 1.067e-05 1.0496e-05 1.0475e-05 1.0499e-05 1.0539e-05 1.0544e-05 1.0575e-05 1.0539e-05 1.058e-05 1.0572e-05 1.0632e-05 1.0662e-05 1.0576e-05 1.0603e-05 1.0485e-05 1.0559e-05 1.0625e-05 1.0561e-05 1.0521e-05 1.0556e-05 1.0563e-05 1.0552e-05 1.0518e-05 1.0484e-05 1.051e-05 1.0564e-05 1.0645e-05 1.0522e-05 1.0587e-05 1.055e-05 1.0528e-05 1.0578e-05 1.0553e-05 1.0541e-05 1.0489e-05 1.054e-05 1.0692e-05 1.0586e-05 1.0536e-05 1.0527e-05 1.0521e-05 1.0466e-05 1.056e-05 1.0326e-321 +gather: 2 (ozark): 0.0010499: breakdown: 1.2737e-05 1.1093e-05 1.084e-05 1.071e-05 1.0682e-05 1.0697e-05 1.068e-05 1.0594e-05 1.0543e-05 1.0564e-05 1.0641e-05 1.3895e-05 1.0677e-05 1.0648e-05 1.0611e-05 1.053e-05 1.0576e-05 1.0542e-05 1.0563e-05 1.0571e-05 1.0584e-05 1.0496e-05 1.0561e-05 1.0516e-05 1.0504e-05 1.0598e-05 1.049e-05 1.0543e-05 1.0564e-05 1.055e-05 1.0487e-05 1.0514e-05 1.0499e-05 1.0573e-05 1.0565e-05 1.054e-05 1.0576e-05 1.0574e-05 1.0548e-05 1.0575e-05 1.0544e-05 1.0572e-05 1.0517e-05 1.0454e-05 1.0595e-05 1.0503e-05 1.0415e-05 1.0499e-05 1.0466e-05 1.049e-05 1.0472e-05 1.0524e-05 1.0504e-05 1.0553e-05 1.0536e-05 1.0523e-05 1.05e-05 1.0514e-05 1.0493e-05 1.0603e-05 1.061e-05 1.0506e-05 1.0595e-05 1.0495e-05 1.0472e-05 1.0542e-05 1.0515e-05 1.0502e-05 1.055e-05 1.0528e-05 1.0521e-05 1.0463e-05 1.0536e-05 1.0514e-05 1.0562e-05 1.0451e-05 1.0563e-05 1.0509e-05 1.0514e-05 1.0589e-05 1.0515e-05 1.0495e-05 1.0509e-05 1.0523e-05 1.0461e-05 1.0514e-05 1.0481e-05 1.0469e-05 1.0566e-05 1.0594e-05 1.0481e-05 1.0552e-05 1.055e-05 1.0549e-05 1.0544e-05 1.0493e-05 1.0461e-05 1.0583e-05 1.0495e-05 1.0326e-321 +gather: 3 (ozark): 0.00105296: breakdown: 1.2346e-05 1.1251e-05 1.0981e-05 1.0918e-05 1.0768e-05 1.0651e-05 1.0801e-05 1.0648e-05 1.0674e-05 1.0583e-05 1.054e-05 1.4257e-05 1.0916e-05 1.0637e-05 1.0546e-05 1.0602e-05 1.0595e-05 1.0568e-05 1.0677e-05 1.0658e-05 1.0597e-05 1.0599e-05 1.0588e-05 1.0549e-05 1.0599e-05 1.0492e-05 1.0531e-05 1.0589e-05 1.0547e-05 1.0585e-05 1.0636e-05 1.049e-05 1.0568e-05 1.0574e-05 1.0561e-05 1.0567e-05 1.0551e-05 1.0584e-05 1.0553e-05 1.0523e-05 1.053e-05 1.0584e-05 1.0539e-05 1.0521e-05 1.0567e-05 1.0472e-05 1.0534e-05 1.0611e-05 1.0491e-05 1.0504e-05 1.0609e-05 1.0556e-05 1.0494e-05 1.0544e-05 1.0523e-05 1.0557e-05 1.0548e-05 1.0546e-05 1.0564e-05 1.0585e-05 1.0552e-05 1.0591e-05 1.0507e-05 1.0513e-05 1.0521e-05 1.0524e-05 1.0575e-05 1.054e-05 1.063e-05 1.0557e-05 1.056e-05 1.0516e-05 1.0524e-05 1.0612e-05 1.0487e-05 1.0491e-05 1.0566e-05 1.0514e-05 1.0542e-05 1.0687e-05 1.0513e-05 1.0495e-05 1.054e-05 1.0484e-05 1.0528e-05 1.0496e-05 1.0573e-05 1.0505e-05 1.0566e-05 1.0461e-05 1.0563e-05 1.0576e-05 1.0481e-05 1.0578e-05 1.0549e-05 1.052e-05 1.0666e-05 1.0529e-05 1.0545e-05 1.0326e-321 + +dpotrf_double +gather: 0 (ozark): 3.568e-05: breakdown: 2.039e-06 7.06e-07 4.16e-07 3.82e-07 3.66e-07 3.88e-07 3.83e-07 3.62e-07 3.52e-07 3.43e-07 3.42e-07 3.4e-07 3.37e-07 3.33e-07 3.47e-07 3.35e-07 3.4e-07 3.54e-07 3.43e-07 3.46e-07 3.47e-07 3.41e-07 3.38e-07 3.36e-07 3.36e-07 3.37e-07 3.34e-07 3.39e-07 3.37e-07 3.43e-07 3.4e-07 3.44e-07 3.41e-07 3.37e-07 3.35e-07 3.39e-07 3.4e-07 3.34e-07 3.39e-07 3.4e-07 3.32e-07 3.34e-07 3.37e-07 3.39e-07 3.41e-07 3.46e-07 3.35e-07 3.37e-07 3.31e-07 3.36e-07 3.35e-07 3.4e-07 3.87e-07 3.41e-07 3.32e-07 3.34e-07 3.27e-07 3.32e-07 3.22e-07 3.33e-07 3.27e-07 3.33e-07 3.32e-07 3.34e-07 3.38e-07 3.42e-07 3.5e-07 3.37e-07 3.34e-07 3.3e-07 3.31e-07 3.36e-07 3.35e-07 3.38e-07 3.32e-07 3.34e-07 3.31e-07 3.32e-07 3.34e-07 3.31e-07 3.36e-07 3.31e-07 3.33e-07 3.29e-07 3.3e-07 3.34e-07 3.36e-07 3.36e-07 3.36e-07 3.28e-07 3.34e-07 3.39e-07 3.29e-07 3.34e-07 3.24e-07 3.27e-07 3.27e-07 3.31e-07 3.33e-07 1.9812e-321 +gather: 1 (ozark): 3.6751e-05: breakdown: 1.742e-06 4.88e-07 4.09e-07 3.96e-07 3.98e-07 3.91e-07 4.11e-07 3.68e-07 3.67e-07 3.54e-07 4.01e-07 3.71e-07 3.59e-07 3.65e-07 3.51e-07 3.59e-07 3.56e-07 3.53e-07 3.63e-07 3.46e-07 3.47e-07 3.44e-07 3.47e-07 3.58e-07 3.49e-07 3.4e-07 3.45e-07 3.48e-07 3.45e-07 3.5e-07 3.5e-07 3.46e-07 3.52e-07 3.81e-07 3.59e-07 3.53e-07 3.53e-07 3.72e-07 3.43e-07 3.56e-07 3.4e-07 3.81e-07 3.54e-07 3.55e-07 3.49e-07 3.5e-07 3.49e-07 3.54e-07 3.51e-07 3.5e-07 3.49e-07 3.55e-07 3.55e-07 3.49e-07 3.49e-07 3.55e-07 3.47e-07 3.57e-07 3.56e-07 3.6e-07 3.44e-07 3.55e-07 3.53e-07 3.5e-07 3.49e-07 3.81e-07 3.63e-07 3.67e-07 3.57e-07 3.57e-07 3.42e-07 3.44e-07 3.54e-07 3.46e-07 3.6e-07 3.45e-07 3.85e-07 3.48e-07 3.57e-07 3.54e-07 3.58e-07 3.57e-07 3.45e-07 3.45e-07 3.47e-07 3.44e-07 3.45e-07 3.41e-07 3.43e-07 3.39e-07 3.48e-07 3.5e-07 3.44e-07 3.45e-07 3.42e-07 3.47e-07 3.56e-07 3.49e-07 3.44e-07 2.77171e-321 +gather: 2 (ozark): 3.607e-05: breakdown: 2.04e-06 6.22e-07 4.35e-07 3.89e-07 3.73e-07 3.76e-07 3.6e-07 3.49e-07 3.45e-07 3.47e-07 3.51e-07 3.47e-07 3.36e-07 3.39e-07 3.4e-07 3.39e-07 3.4e-07 3.4e-07 3.45e-07 3.38e-07 3.43e-07 3.48e-07 3.43e-07 3.38e-07 3.41e-07 3.44e-07 3.38e-07 3.47e-07 3.44e-07 3.4e-07 3.35e-07 3.43e-07 3.44e-07 4.92e-07 3.46e-07 3.49e-07 3.45e-07 3.48e-07 3.31e-07 3.45e-07 3.4e-07 3.47e-07 3.36e-07 3.39e-07 3.4e-07 3.35e-07 3.39e-07 3.38e-07 3.38e-07 3.38e-07 3.38e-07 3.38e-07 3.37e-07 3.35e-07 3.73e-07 3.42e-07 3.36e-07 3.4e-07 3.41e-07 3.4e-07 3.31e-07 3.34e-07 3.4e-07 3.36e-07 3.37e-07 3.49e-07 3.46e-07 3.39e-07 3.38e-07 3.32e-07 3.37e-07 3.45e-07 3.41e-07 3.39e-07 3.42e-07 3.3e-07 3.37e-07 3.4e-07 3.38e-07 3.4e-07 3.42e-07 3.41e-07 3.38e-07 3.34e-07 3.4e-07 3.41e-07 3.34e-07 3.39e-07 3.42e-07 3.36e-07 3.34e-07 3.38e-07 3.35e-07 3.36e-07 3.37e-07 3.37e-07 3.44e-07 3.47e-07 3.39e-07 1.58595e-321 +gather: 3 (ozark): 3.5959e-05: breakdown: 2.109e-06 6.3e-07 3.87e-07 3.83e-07 3.79e-07 3.71e-07 4.03e-07 3.65e-07 3.66e-07 3.51e-07 3.48e-07 3.36e-07 3.41e-07 3.4e-07 3.4e-07 3.35e-07 3.34e-07 3.47e-07 3.41e-07 3.41e-07 3.35e-07 3.37e-07 3.38e-07 3.47e-07 3.4e-07 3.35e-07 3.38e-07 3.41e-07 3.49e-07 3.44e-07 3.34e-07 3.46e-07 3.42e-07 3.49e-07 3.42e-07 3.43e-07 3.35e-07 3.4e-07 3.4e-07 3.42e-07 3.4e-07 3.4e-07 3.44e-07 3.36e-07 3.36e-07 3.39e-07 3.49e-07 3.4e-07 3.36e-07 3.37e-07 3.37e-07 3.43e-07 3.36e-07 3.41e-07 3.45e-07 3.35e-07 3.46e-07 3.41e-07 3.47e-07 3.3e-07 3.33e-07 3.29e-07 3.36e-07 3.29e-07 3.37e-07 3.48e-07 3.55e-07 3.49e-07 3.39e-07 3.35e-07 3.43e-07 3.41e-07 3.39e-07 3.71e-07 3.4e-07 3.36e-07 3.32e-07 3.33e-07 3.33e-07 3.39e-07 3.34e-07 3.33e-07 3.3e-07 3.31e-07 3.36e-07 3.34e-07 3.31e-07 3.4e-07 3.31e-07 3.37e-07 3.4e-07 3.36e-07 3.39e-07 3.94e-07 3.33e-07 3.35e-07 3.3e-07 3.29e-07 3.27e-07 1.42785e-321 + +level2_complex +gather: 0 (ozark): 0.000867368: breakdown: 9.145e-06 8.915e-06 8.772e-06 8.76e-06 8.768e-06 8.72e-06 8.704e-06 8.679e-06 8.683e-06 8.686e-06 8.718e-06 8.67e-06 8.676e-06 8.661e-06 8.634e-06 8.686e-06 8.673e-06 8.63e-06 8.678e-06 8.659e-06 8.658e-06 8.706e-06 8.667e-06 8.679e-06 8.648e-06 8.653e-06 8.681e-06 8.668e-06 8.626e-06 8.665e-06 8.676e-06 8.659e-06 8.678e-06 8.721e-06 8.667e-06 8.638e-06 8.675e-06 8.742e-06 8.683e-06 8.698e-06 8.697e-06 8.687e-06 8.664e-06 8.672e-06 8.674e-06 8.689e-06 8.687e-06 8.671e-06 8.697e-06 8.724e-06 8.717e-06 8.711e-06 8.652e-06 8.673e-06 8.694e-06 8.702e-06 8.654e-06 8.661e-06 8.681e-06 8.687e-06 8.643e-06 8.712e-06 8.703e-06 8.643e-06 8.705e-06 8.67e-06 8.707e-06 8.654e-06 8.652e-06 8.72e-06 8.698e-06 8.676e-06 8.698e-06 8.713e-06 8.691e-06 8.744e-06 8.718e-06 8.691e-06 8.697e-06 8.709e-06 8.734e-06 8.669e-06 8.674e-06 8.694e-06 8.694e-06 8.635e-06 8.699e-06 8.694e-06 8.679e-06 8.667e-06 8.736e-06 8.684e-06 8.71e-06 8.697e-06 8.692e-06 8.701e-06 1.5241e-05 8.896e-06 8.629e-06 3.95747e-321 +gather: 1 (ozark): 0.000860855: breakdown: 9.106e-06 8.78e-06 8.755e-06 8.846e-06 8.663e-06 8.741e-06 8.653e-06 8.635e-06 8.666e-06 8.691e-06 8.608e-06 8.592e-06 8.711e-06 8.644e-06 8.571e-06 8.586e-06 8.654e-06 8.64e-06 8.55e-06 8.627e-06 8.656e-06 8.59e-06 8.635e-06 8.671e-06 8.584e-06 8.598e-06 8.646e-06 8.627e-06 8.597e-06 8.646e-06 8.615e-06 8.643e-06 8.589e-06 8.61e-06 8.627e-06 8.598e-06 8.589e-06 8.598e-06 8.625e-06 8.616e-06 8.58e-06 8.614e-06 8.611e-06 8.636e-06 8.597e-06 8.644e-06 8.629e-06 8.579e-06 8.68e-06 8.624e-06 8.615e-06 8.63e-06 8.625e-06 8.603e-06 8.65e-06 8.614e-06 8.63e-06 8.621e-06 8.618e-06 8.62e-06 8.651e-06 8.595e-06 8.625e-06 8.644e-06 8.623e-06 8.617e-06 8.631e-06 8.638e-06 8.58e-06 8.587e-06 8.651e-06 8.603e-06 8.565e-06 8.648e-06 8.621e-06 8.597e-06 8.617e-06 8.639e-06 8.629e-06 8.615e-06 8.588e-06 8.593e-06 8.601e-06 8.571e-06 8.602e-06 8.622e-06 8.59e-06 8.631e-06 8.654e-06 8.579e-06 8.589e-06 8.649e-06 8.628e-06 8.588e-06 8.587e-06 8.612e-06 8.575e-06 1.482e-05 8.901e-06 2.37646e-321 +gather: 2 (ozark): 0.000864709: breakdown: 8.987e-06 8.674e-06 8.645e-06 8.716e-06 8.76e-06 8.651e-06 8.673e-06 8.695e-06 8.67e-06 8.732e-06 8.645e-06 8.696e-06 8.743e-06 8.691e-06 8.679e-06 8.688e-06 8.682e-06 8.605e-06 8.626e-06 8.735e-06 8.702e-06 8.604e-06 8.672e-06 8.669e-06 8.716e-06 8.626e-06 8.684e-06 8.623e-06 8.592e-06 8.662e-06 8.67e-06 8.636e-06 8.63e-06 8.716e-06 8.669e-06 8.67e-06 8.649e-06 8.655e-06 8.649e-06 8.683e-06 8.69e-06 8.663e-06 8.676e-06 8.646e-06 8.702e-06 8.708e-06 8.636e-06 8.66e-06 8.739e-06 8.644e-06 8.669e-06 8.653e-06 8.649e-06 8.676e-06 8.652e-06 8.705e-06 8.695e-06 8.639e-06 8.679e-06 8.711e-06 8.675e-06 8.629e-06 8.69e-06 8.706e-06 8.638e-06 8.724e-06 8.67e-06 8.625e-06 8.662e-06 8.703e-06 8.672e-06 8.631e-06 8.68e-06 8.655e-06 8.655e-06 8.675e-06 8.651e-06 8.645e-06 8.677e-06 8.631e-06 8.656e-06 8.649e-06 8.672e-06 8.682e-06 8.639e-06 8.614e-06 8.628e-06 8.62e-06 8.681e-06 8.702e-06 8.663e-06 8.704e-06 8.722e-06 8.598e-06 8.653e-06 8.687e-06 1.4641e-05 8.854e-06 8.688e-06 2.45551e-321 +gather: 3 (ozark): 0.000864736: breakdown: 9.177e-06 8.76e-06 8.779e-06 8.695e-06 8.729e-06 8.753e-06 8.759e-06 8.691e-06 8.721e-06 8.666e-06 8.654e-06 8.587e-06 8.643e-06 8.645e-06 8.616e-06 8.648e-06 8.685e-06 8.63e-06 8.574e-06 8.721e-06 8.707e-06 8.637e-06 8.635e-06 8.636e-06 8.661e-06 8.712e-06 8.661e-06 8.628e-06 8.617e-06 8.63e-06 8.624e-06 8.713e-06 8.659e-06 8.655e-06 8.651e-06 8.689e-06 8.58e-06 8.691e-06 8.696e-06 8.583e-06 8.686e-06 8.65e-06 8.652e-06 8.624e-06 8.702e-06 8.657e-06 8.633e-06 8.644e-06 8.661e-06 8.669e-06 8.635e-06 8.668e-06 8.657e-06 8.663e-06 8.642e-06 8.648e-06 8.68e-06 8.659e-06 8.668e-06 8.623e-06 8.62e-06 8.649e-06 8.671e-06 8.666e-06 8.635e-06 8.721e-06 8.72e-06 8.652e-06 8.632e-06 8.694e-06 8.579e-06 8.617e-06 8.682e-06 8.633e-06 8.665e-06 8.673e-06 8.681e-06 8.67e-06 8.626e-06 8.644e-06 8.658e-06 8.68e-06 8.592e-06 8.689e-06 8.693e-06 8.642e-06 8.807e-06 8.682e-06 8.685e-06 8.687e-06 8.661e-06 8.68e-06 8.659e-06 8.618e-06 8.682e-06 8.731e-06 1.4807e-05 8.935e-06 8.699e-06 1.9812e-321 + +level3_complex +gather: 0 (ozark): 0.0538252: breakdown: 0.00055978 0.000543486 0.000548978 0.000548229 0.000538536 0.00054722 0.000543739 0.000546811 0.00053979 0.000550027 0.000541862 0.000547632 0.000543472 0.000545303 0.000549762 0.000537851 0.000549892 0.000544085 0.00054895 0.000540855 0.000548848 0.000540299 0.000544242 0.000545852 0.000549127 0.000548827 0.000535735 0.000547262 0.000539781 0.000546898 0.000543729 0.000547786 0.000540317 0.000545381 0.00054073 0.000546491 0.000550097 0.000535818 0.000547913 0.000542254 0.000547146 0.000539271 0.000545408 0.000541239 0.000543001 0.000539823 0.000547653 0.000539921 0.000539413 0.000546936 0.000542424 0.000548745 0.000539712 0.000545207 0.00053827 0.000543972 0.000539954 0.000543193 0.000537575 0.000543291 0.000545693 0.000536742 0.000546458 0.000540569 0.000545824 0.000540406 0.00054601 0.000539742 0.000545462 0.000539682 0.000546458 0.000546932 0.000537796 0.000546188 0.000541625 0.000547296 0.000540232 0.000546755 0.000540039 0.000547451 0.000539621 0.000546369 0.000539216 0.000529485 0.000546832 0.000540062 0.000547439 0.00054006 0.000545989 0.000539237 0.000542366 0.000539015 0.000547242 0.00054166 0.000542921 0.00054906 0.000535962 0.000549207 0.000540252 3.95747e-321 +gather: 1 (ozark): 0.0536701: breakdown: 0.000551591 0.000541947 0.000545451 0.000540808 0.000539546 0.000538551 0.000535682 0.00054463 0.000541579 0.000547235 0.00054549 0.000546029 0.00054306 0.00054624 0.000556297 0.000542625 0.000546403 0.000537593 0.000543726 0.000538994 0.000544698 0.000537991 0.000546843 0.000544523 0.000545878 0.00053929 0.000542762 0.000542064 0.000539548 0.000542867 0.00054179 0.000544508 0.000541368 0.000542865 0.000540428 0.000547096 0.000539224 0.000544651 0.000544216 0.000541226 0.000542148 0.000535662 0.000543039 0.000539396 0.000543314 0.000539253 0.000544487 0.000541574 0.000543724 0.000541306 0.000543121 0.000542549 0.000538428 0.00054271 0.000539764 0.000543492 0.000539096 0.000542387 0.000538867 0.000543126 0.000538717 0.000545024 0.00054263 0.000541388 0.000543049 0.000538711 0.000543948 0.000539657 0.00054413 0.00054032 0.000544341 0.000539652 0.000542949 0.00053813 0.000535218 0.000544154 0.000538467 0.000544665 0.000539001 0.000543345 0.000540691 0.000544732 0.000540622 0.000543947 0.000539468 0.000542463 0.000541501 0.000539309 0.000543069 0.000539667 0.000542574 0.000540279 0.000541807 0.000538951 0.000542046 0.000540192 0.000543317 0.000538987 0.000544213 4.79244e-322 +gather: 2 (ozark): 0.0538132: breakdown: 0.000550435 0.000543538 0.000543608 0.000544867 0.000542774 0.000545064 0.000541664 0.000545466 0.00054213 0.000548189 0.000546855 0.000554111 0.00055014 0.000559412 0.000565018 0.000545773 0.000552451 0.000543912 0.000545019 0.000539704 0.000548052 0.00054387 0.000552129 0.000549555 0.000548878 0.00054612 0.000542935 0.000546008 0.00054213 0.000545904 0.000541016 0.000545195 0.000539482 0.00054378 0.000539897 0.000542868 0.000546328 0.00054018 0.000544794 0.00054101 0.000543033 0.000539296 0.000544094 0.000538696 0.000544314 0.000540135 0.000542899 0.000546352 0.000543747 0.000544616 0.000539565 0.000543317 0.000539814 0.00054342 0.000539889 0.000543541 0.000540198 0.000543135 0.000540078 0.000544327 0.000545461 0.000539041 0.000541831 0.000539382 0.00054368 0.000538416 0.000543312 0.000539693 0.000544544 0.000540075 0.000543954 0.000544466 0.00053954 0.000546691 0.000541713 0.000547033 0.000539681 0.000543691 0.000539861 0.00054396 0.000540161 0.000542779 0.000542461 0.000538959 0.000542376 0.000538927 0.000541615 0.000538395 0.000543368 0.000538634 0.000542518 0.000539326 0.000541919 0.000538865 0.000543365 0.000544291 0.000541156 0.000544176 0.000539108 4.79244e-322 +gather: 3 (ozark): 0.0535708: breakdown: 0.000547376 0.000542838 0.000543107 0.000542089 0.000538403 0.000543558 0.000539586 0.000543657 0.000540671 0.000546131 0.00054811 0.000554863 0.000550522 0.000559185 0.000566595 0.000543105 0.000550561 0.000545429 0.000547224 0.000541117 0.000545292 0.000539994 0.000550631 0.000543468 0.000544843 0.000542488 0.000533926 0.000541398 0.000537941 0.000539308 0.00053623 0.000538794 0.00053757 0.000540321 0.000536797 0.000541441 0.000536323 0.000540603 0.000540146 0.000539231 0.00054142 0.000531915 0.000540206 0.000538299 0.000540316 0.000536559 0.000540995 0.000537689 0.000541769 0.000536105 0.000540842 0.000541265 0.000537981 0.000540714 0.000537244 0.000540758 0.000537241 0.000541429 0.000538617 0.000541695 0.000537148 0.000541746 0.000537491 0.000540748 0.000541341 0.000536701 0.00054033 0.000538023 0.0005418 0.000537335 0.000539515 0.000538278 0.000543444 0.000538244 0.000544648 0.000543962 0.000539196 0.000544652 0.000536673 0.000539884 0.000538555 0.000541914 0.000537178 0.000540089 0.00053624 0.000538286 0.000537264 0.000540487 0.000541481 0.000537355 0.00054157 0.000535785 0.000541459 0.000539439 0.000542353 0.000537141 0.000541068 0.000537104 0.000540902 4.79244e-322 + +level2_double +gather: 0 (ozark): 0.000378346: breakdown: 8.035e-06 3.892e-06 3.828e-06 3.807e-06 3.806e-06 3.811e-06 3.796e-06 3.784e-06 3.804e-06 3.777e-06 3.773e-06 3.791e-06 3.79e-06 3.75e-06 3.767e-06 3.774e-06 3.784e-06 3.793e-06 3.791e-06 3.765e-06 3.795e-06 3.776e-06 3.779e-06 3.766e-06 3.76e-06 3.789e-06 3.78e-06 3.762e-06 3.76e-06 3.774e-06 3.84e-06 3.807e-06 3.785e-06 3.788e-06 3.769e-06 3.757e-06 3.792e-06 3.79e-06 3.774e-06 3.807e-06 3.767e-06 3.77e-06 3.779e-06 3.75e-06 3.75e-06 3.76e-06 3.751e-06 3.77e-06 3.76e-06 3.781e-06 3.756e-06 3.776e-06 3.8e-06 3.796e-06 3.79e-06 3.783e-06 3.763e-06 3.786e-06 3.791e-06 3.744e-06 3.761e-06 3.755e-06 3.776e-06 3.754e-06 3.774e-06 3.755e-06 3.76e-06 3.757e-06 3.746e-06 3.771e-06 3.779e-06 3.753e-06 3.778e-06 3.749e-06 3.795e-06 3.826e-06 3.776e-06 3.744e-06 3.748e-06 3.781e-06 3.782e-06 3.773e-06 3.786e-06 3.776e-06 3.76e-06 3.759e-06 3.784e-06 3.798e-06 3.787e-06 3.782e-06 3.793e-06 3.771e-06 3.776e-06 3.742e-06 3.767e-06 3.777e-06 3.787e-06 3.812e-06 3.805e-06 4.79244e-322 +gather: 1 (ozark): 0.000377205: breakdown: 7.512e-06 3.92e-06 3.818e-06 3.831e-06 3.816e-06 3.802e-06 3.804e-06 3.793e-06 3.782e-06 3.817e-06 3.77e-06 3.767e-06 3.777e-06 3.8e-06 3.757e-06 3.766e-06 3.769e-06 3.796e-06 3.766e-06 3.754e-06 3.79e-06 3.754e-06 3.808e-06 3.78e-06 3.735e-06 3.778e-06 3.776e-06 3.766e-06 3.749e-06 3.779e-06 3.777e-06 3.77e-06 3.744e-06 3.791e-06 3.753e-06 3.794e-06 3.771e-06 3.729e-06 3.773e-06 3.763e-06 3.76e-06 3.774e-06 3.75e-06 3.754e-06 3.756e-06 3.755e-06 3.769e-06 3.778e-06 3.772e-06 3.742e-06 3.741e-06 3.778e-06 3.748e-06 3.851e-06 3.782e-06 3.741e-06 3.736e-06 3.777e-06 3.781e-06 3.765e-06 3.762e-06 3.772e-06 3.777e-06 3.787e-06 3.752e-06 3.772e-06 3.786e-06 3.761e-06 3.761e-06 3.809e-06 3.719e-06 3.765e-06 3.734e-06 3.81e-06 3.773e-06 3.752e-06 3.769e-06 3.789e-06 3.782e-06 3.746e-06 3.751e-06 3.778e-06 3.749e-06 3.769e-06 3.755e-06 3.786e-06 3.757e-06 3.777e-06 3.768e-06 3.769e-06 3.769e-06 3.769e-06 3.771e-06 3.762e-06 3.769e-06 3.758e-06 3.737e-06 3.752e-06 3.774e-06 4.79244e-322 +gather: 2 (ozark): 0.000375657: breakdown: 7.273e-06 3.933e-06 3.875e-06 3.833e-06 3.842e-06 3.807e-06 3.773e-06 3.807e-06 3.793e-06 3.766e-06 3.74e-06 3.735e-06 3.774e-06 3.789e-06 3.737e-06 3.752e-06 3.732e-06 3.796e-06 3.768e-06 3.755e-06 3.734e-06 3.734e-06 3.756e-06 3.768e-06 3.773e-06 3.754e-06 3.742e-06 3.75e-06 3.743e-06 3.727e-06 3.757e-06 3.781e-06 3.729e-06 3.754e-06 3.76e-06 3.755e-06 3.724e-06 3.766e-06 3.751e-06 3.728e-06 3.796e-06 3.748e-06 3.747e-06 3.735e-06 3.742e-06 3.782e-06 3.746e-06 3.751e-06 3.764e-06 3.754e-06 3.75e-06 3.755e-06 3.739e-06 3.748e-06 3.749e-06 3.751e-06 3.75e-06 3.779e-06 3.733e-06 3.727e-06 3.738e-06 3.728e-06 3.79e-06 3.781e-06 3.757e-06 3.749e-06 3.725e-06 3.738e-06 3.753e-06 3.738e-06 3.755e-06 3.751e-06 3.756e-06 3.766e-06 3.729e-06 3.771e-06 3.748e-06 3.742e-06 3.748e-06 3.737e-06 3.763e-06 3.746e-06 3.758e-06 3.763e-06 3.738e-06 3.714e-06 3.743e-06 3.75e-06 3.779e-06 3.761e-06 3.739e-06 3.782e-06 3.735e-06 3.732e-06 3.785e-06 3.781e-06 3.765e-06 3.773e-06 3.738e-06 4.79244e-322 +gather: 3 (ozark): 0.000377028: breakdown: 6.824e-06 3.871e-06 3.843e-06 3.854e-06 3.794e-06 3.767e-06 3.777e-06 3.774e-06 3.775e-06 3.784e-06 3.82e-06 3.761e-06 3.798e-06 3.75e-06 3.768e-06 3.767e-06 3.779e-06 3.788e-06 3.797e-06 3.783e-06 3.823e-06 3.782e-06 3.818e-06 3.782e-06 3.803e-06 3.748e-06 3.767e-06 3.797e-06 3.798e-06 3.713e-06 3.788e-06 3.789e-06 3.737e-06 3.807e-06 3.792e-06 3.732e-06 3.774e-06 3.852e-06 3.785e-06 3.759e-06 3.756e-06 3.753e-06 3.79e-06 3.761e-06 3.749e-06 3.799e-06 3.757e-06 3.769e-06 3.772e-06 3.748e-06 3.805e-06 3.778e-06 3.747e-06 3.738e-06 3.755e-06 3.742e-06 3.797e-06 3.78e-06 3.756e-06 3.757e-06 3.798e-06 3.779e-06 3.762e-06 3.759e-06 3.763e-06 3.813e-06 3.798e-06 3.773e-06 3.767e-06 3.752e-06 3.784e-06 3.742e-06 3.782e-06 3.807e-06 3.798e-06 3.759e-06 3.8e-06 3.738e-06 3.759e-06 3.768e-06 3.781e-06 3.813e-06 3.803e-06 3.752e-06 3.794e-06 3.721e-06 3.749e-06 3.785e-06 3.8e-06 3.785e-06 3.766e-06 3.756e-06 3.771e-06 3.735e-06 3.765e-06 3.767e-06 3.779e-06 3.767e-06 3.809e-06 4.79244e-322 + +level3_double +gather: 0 (ozark): 0.013612: breakdown: 0.000137439 0.000135248 0.000142266 0.00013641 0.00013666 0.000136452 0.000136549 0.000137653 0.000136242 0.000136384 0.000139447 0.000135525 0.000135817 0.000136879 0.000136827 0.000136968 0.0001368 0.000151418 0.000136682 0.000136878 0.000136249 0.00013699 0.000138251 0.000138002 0.000144697 0.000138951 0.000138674 0.000139006 0.000138346 0.000138737 0.000138277 0.000150323 0.000135822 0.00013623 0.000136439 0.000135897 0.000136263 0.00013613 0.000142 0.000134647 0.000135762 0.000135923 0.000135727 0.000135732 0.000135398 0.000136092 0.000140582 0.000135843 0.000136975 0.000136878 0.000136689 0.000137124 0.000137116 0.000142684 0.000135559 0.000136155 0.000136384 0.000136855 0.000137057 0.000137618 0.000142347 0.000135631 0.000136813 0.000137354 0.000136669 0.000137349 0.000136907 0.000136542 0.000143874 0.000135867 0.000135643 0.000136666 0.000136657 0.000136491 0.000136529 0.000139722 0.000136014 0.000136526 0.000136317 0.000136224 0.000136852 0.000135923 0.000142899 0.000135799 0.000135893 0.000136045 0.000136262 0.000136381 0.000136772 0.000140402 0.000135093 0.000135558 0.000135691 0.000135739 0.000135598 0.000135852 0.000135737 0.000145457 0.000135248 7.95446e-322 +gather: 1 (ozark): 0.1136195: breakdown: 0.000138176 0.00013565 0.000143277 0.000136673 0.000136618 0.00013671 0.000136653 0.000136514 0.000137308 0.000136652 0.000142868 0.0001366 0.000137026 0.000136268 0.000136929 0.000136964 0.000138146 0.000142308 0.0001375 0.000137346 0.000137585 0.000138227 0.000137883 0.000137738 0.000142911 0.000138691 0.000137783 0.00013784 0.000138889 0.000138536 0.000138032 0.000143095 0.000136961 0.000137316 0.000137544 0.000137264 0.000136793 0.000136994 0.000140623 0.000137524 0.000136353 0.000137041 0.000136458 0.000137129 0.000136572 0.000137049 0.000140948 0.000134394 0.000135512 0.000136655 0.000136194 0.00013667 0.000137126 0.000141529 0.000137369 0.000136999 0.000136428 0.000136381 0.000136402 0.000136151 0.000141558 0.000134545 0.000136237 0.000136804 0.000136795 0.000137166 0.000137481 0.000137738 0.000141553 0.000137137 0.000136745 0.000136077 0.00013621 0.000136482 0.000136231 0.000140824 0.000136197 0.000136674 0.000135973 0.000136704 0.000136847 0.000136922 0.000141772 0.00013803 0.000136442 0.000136615 0.000136524 0.000136912 0.000136511 0.0001413 0.000134221 0.000136172 0.000136573 0.000136899 0.000136804 0.000137158 0.000136199 0.000143031 0.000135647 4.79244e-322 +gather: 2 (ozark): 0.2136247: breakdown: 0.00013855 0.000136033 0.000141318 0.000136556 0.00013614 0.000137407 0.000137245 0.000137249 0.000136647 0.000137335 0.000142316 0.000137039 0.000136559 0.00013707 0.000137341 0.000137757 0.000137499 0.000143004 0.000137948 0.000138045 0.00013743 0.000137632 0.000137593 0.000137794 0.000142373 0.000138192 0.000137964 0.000139087 0.000138493 0.000138635 0.000138871 0.000142191 0.000138046 0.0001377 0.000137257 0.00013722 0.000137161 0.000136117 0.000142166 0.000136582 0.000136953 0.000137206 0.000137419 0.000137067 0.00013683 0.000136755 0.000141894 0.00013716 0.000136754 0.000136251 0.000136464 0.000136613 0.000137158 0.000141419 0.000137366 0.000136705 0.000136541 0.000137122 0.000136445 0.000137042 0.000141469 0.000136135 0.000137362 0.000137254 0.000136432 0.000137583 0.000136443 0.000137165 0.000139909 0.000135689 0.000137449 0.000136929 0.00013699 0.000136446 0.000137062 0.000138618 0.000136036 0.000136417 0.000137072 0.00013672 0.000137436 0.000137476 0.000139347 0.00013521 0.000136924 0.000136824 0.000137233 0.000136498 0.000136275 0.000139838 0.000136246 0.000136318 0.000136209 0.00013724 0.000136101 0.000136778 0.000135724 0.00014169 0.00013543 4.79244e-322 +gather: 3 (ozark): 0.0136833: breakdown: 0.000137755 0.000137057 0.000161103 0.000137187 0.000137078 0.000136033 0.000136746 0.000136884 0.00013684 0.000141032 0.000136113 0.000136183 0.000136757 0.000137477 0.000137461 0.000137472 0.000137179 0.000175561 0.000137687 0.000137406 0.00013739 0.000137872 0.000137693 0.000138476 0.000142229 0.00013853 0.000138437 0.000138516 0.000139438 0.000138633 0.000138144 0.000142989 0.000139092 0.000137896 0.0001369 0.000137677 0.000137364 0.000136515 0.00014102 0.000137012 0.000136586 0.00013587 0.000136078 0.000136019 0.000136107 0.000141825 0.000136524 0.000136417 0.000135826 0.000136311 0.000136288 0.000136025 0.000136041 0.000150565 0.000136578 0.000136137 0.000136654 0.000136913 0.000137091 0.000136599 0.000141485 0.00013705 0.000136881 0.000136513 0.000137141 0.000136549 0.00013709 0.000143895 0.000136945 0.000136209 0.000136313 0.000136171 0.000136267 0.000136307 0.000140881 0.000135192 0.000136107 0.000135952 0.000136634 0.000136678 0.000136549 0.000136721 0.000141329 0.000137277 0.000136741 0.000136888 0.000136578 0.000136789 0.000136383 0.0001411 0.000137554 0.000136821 0.000137963 0.000135352 0.0001367 0.000136426 0.000141386 0.000135826 0.000137355 4.79244e-322 + +level1_complex +gather: 0 (ozark): 1.8592e-05: breakdown: 4.16e-07 2.55e-07 2.24e-07 2.26e-07 2.23e-07 2.32e-07 2.11e-07 2.02e-07 1.83e-07 2.15e-07 2.01e-07 1.83e-07 1.83e-07 1.83e-07 1.86e-07 1.82e-07 1.82e-07 1.81e-07 1.82e-07 1.83e-07 1.81e-07 1.81e-07 1.81e-07 1.82e-07 1.83e-07 1.83e-07 1.84e-07 1.83e-07 1.81e-07 1.8e-07 1.82e-07 1.81e-07 1.81e-07 1.81e-07 1.81e-07 1.81e-07 1.82e-07 2.31e-07 1.81e-07 1.82e-07 1.8e-07 1.8e-07 1.8e-07 1.84e-07 1.82e-07 1.8e-07 1.82e-07 1.81e-07 1.81e-07 1.8e-07 1.81e-07 1.8e-07 1.81e-07 1.81e-07 1.8e-07 1.81e-07 1.8e-07 1.82e-07 1.81e-07 1.79e-07 1.83e-07 1.8e-07 1.83e-07 1.8e-07 1.82e-07 1.8e-07 1.81e-07 1.79e-07 1.81e-07 1.82e-07 1.8e-07 1.82e-07 1.81e-07 1.8e-07 1.82e-07 1.8e-07 1.79e-07 1.81e-07 1.82e-07 1.82e-07 1.82e-07 1.82e-07 1.8e-07 1.83e-07 1.81e-07 1.8e-07 1.81e-07 1.8e-07 1.81e-07 1.81e-07 1.82e-07 1.8e-07 1.8e-07 1.82e-07 1.81e-07 1.81e-07 1.81e-07 1.82e-07 1.81e-07 4.79244e-322 +gather: 1 (ozark): 2.1363e-05: breakdown: 4.54e-07 2.88e-07 2.6e-07 2.71e-07 2.59e-07 2.53e-07 2.43e-07 2.35e-07 2.34e-07 2.2e-07 3e-07 2.58e-07 2.11e-07 2.12e-07 2.13e-07 2.1e-07 2.11e-07 2.09e-07 2.09e-07 2.1e-07 2.1e-07 2.08e-07 2.07e-07 2.07e-07 2.07e-07 2.07e-07 2.08e-07 2.07e-07 2.06e-07 2.08e-07 2.08e-07 2.06e-07 2.07e-07 2.08e-07 2.06e-07 2.07e-07 2.08e-07 2.06e-07 2.07e-07 2.05e-07 2.09e-07 2.09e-07 2.08e-07 2.08e-07 2.08e-07 2.05e-07 2.07e-07 2.07e-07 2.06e-07 2.05e-07 2.07e-07 2.07e-07 2.06e-07 2.1e-07 2.06e-07 2.06e-07 2.09e-07 2.08e-07 2.08e-07 2.06e-07 2.07e-07 2.09e-07 2.05e-07 2.08e-07 2.09e-07 2.09e-07 2.07e-07 2.06e-07 2.08e-07 2.07e-07 2.08e-07 2.08e-07 2.05e-07 2.09e-07 2.06e-07 2.07e-07 2.06e-07 2.07e-07 2.06e-07 2.06e-07 2.07e-07 2.06e-07 2.06e-07 2.07e-07 2.06e-07 2.5e-07 2.06e-07 2.08e-07 2.09e-07 2.06e-07 2.06e-07 2.08e-07 2.07e-07 2.11e-07 2.06e-07 2.04e-07 2.07e-07 2.04e-07 2.08e-07 4.79244e-322 +gather: 2 (ozark): 1.9089e-05: breakdown: 4.77e-07 3.16e-07 2.44e-07 2.26e-07 2.44e-07 2.19e-07 2.55e-07 2.03e-07 2.04e-07 1.97e-07 1.87e-07 1.82e-07 1.86e-07 1.81e-07 1.83e-07 1.86e-07 1.88e-07 1.86e-07 2.21e-07 1.82e-07 1.84e-07 1.84e-07 1.81e-07 1.82e-07 1.84e-07 1.84e-07 1.84e-07 1.84e-07 1.83e-07 1.83e-07 1.84e-07 1.82e-07 1.83e-07 1.83e-07 1.82e-07 1.85e-07 1.91e-07 1.83e-07 1.88e-07 1.84e-07 1.82e-07 1.87e-07 1.83e-07 1.8e-07 1.87e-07 1.82e-07 1.85e-07 1.87e-07 1.83e-07 1.85e-07 1.85e-07 1.82e-07 1.83e-07 2.47e-07 1.84e-07 1.82e-07 1.86e-07 1.83e-07 1.85e-07 1.85e-07 1.8e-07 1.84e-07 1.83e-07 1.83e-07 1.83e-07 1.86e-07 1.85e-07 1.84e-07 1.86e-07 1.85e-07 1.85e-07 1.85e-07 1.85e-07 1.85e-07 1.86e-07 1.84e-07 1.83e-07 1.84e-07 1.83e-07 1.85e-07 1.86e-07 1.85e-07 1.84e-07 1.85e-07 1.83e-07 1.86e-07 1.89e-07 1.85e-07 1.86e-07 1.83e-07 1.86e-07 1.85e-07 1.84e-07 1.85e-07 1.86e-07 1.84e-07 1.86e-07 1.83e-07 1.86e-07 4.79244e-322 +gather: 3 (ozark): 1.8519e-05: breakdown: 4.53e-07 2.44e-07 2.29e-07 2.29e-07 2.26e-07 2.15e-07 1.97e-07 1.82e-07 1.85e-07 1.8e-07 1.83e-07 1.79e-07 1.81e-07 1.8e-07 1.84e-07 1.8e-07 1.79e-07 1.79e-07 1.77e-07 1.8e-07 1.8e-07 1.84e-07 1.8e-07 1.81e-07 1.8e-07 1.81e-07 1.82e-07 1.8e-07 1.81e-07 1.79e-07 1.8e-07 1.8e-07 1.79e-07 1.8e-07 1.8e-07 1.82e-07 1.82e-07 1.8e-07 1.8e-07 1.8e-07 1.79e-07 1.81e-07 1.82e-07 1.81e-07 1.82e-07 1.8e-07 1.8e-07 1.79e-07 1.79e-07 1.8e-07 1.79e-07 1.82e-07 1.83e-07 1.79e-07 1.86e-07 1.79e-07 1.8e-07 1.82e-07 1.82e-07 1.83e-07 1.82e-07 1.82e-07 1.79e-07 1.82e-07 1.78e-07 1.81e-07 1.78e-07 1.79e-07 1.8e-07 1.8e-07 1.78e-07 2.41e-07 1.8e-07 1.8e-07 1.8e-07 1.81e-07 1.83e-07 1.79e-07 1.81e-07 1.84e-07 1.81e-07 1.78e-07 1.78e-07 1.8e-07 1.83e-07 1.82e-07 1.8e-07 1.79e-07 1.8e-07 1.79e-07 1.79e-07 1.82e-07 1.81e-07 1.8e-07 1.8e-07 1.8e-07 1.81e-07 2.38e-07 1.79e-07 4.79244e-322 + +level1_double +gather: 0 (ozark): 2.6729e-05: breakdown: 1.212e-06 3.51e-07 3.53e-07 3.26e-07 3.21e-07 2.95e-07 2.81e-07 3.3e-07 2.77e-07 3.08e-07 2.66e-07 2.67e-07 2.89e-07 2.58e-07 2.58e-07 2.92e-07 2.49e-07 3.38e-07 2.67e-07 2.56e-07 2.77e-07 2.55e-07 2.55e-07 3.01e-07 2.43e-07 2.43e-07 2.35e-07 2.57e-07 2.64e-07 2.48e-07 2.42e-07 2.48e-07 2.88e-07 2.43e-07 2.39e-07 2.47e-07 2.53e-07 2.46e-07 2.45e-07 2.39e-07 2.88e-07 2.99e-07 2.38e-07 2.41e-07 2.46e-07 2.76e-07 2.43e-07 2.46e-07 2.42e-07 2.42e-07 2.76e-07 2.41e-07 2.45e-07 2.53e-07 2.32e-07 2.32e-07 2.4e-07 2.28e-07 2.38e-07 2.78e-07 2.65e-07 2.4e-07 2.41e-07 2.46e-07 2.58e-07 2.66e-07 2.62e-07 2.96e-07 2.5e-07 2.45e-07 2.46e-07 2.66e-07 2.45e-07 2.35e-07 2.33e-07 2.33e-07 2.97e-07 2.42e-07 2.38e-07 2.49e-07 2.55e-07 2.61e-07 2.39e-07 2.41e-07 2.43e-07 2.99e-07 2.48e-07 2.41e-07 2.46e-07 2.61e-07 2.63e-07 2.44e-07 2.33e-07 2.42e-07 2.78e-07 2.49e-07 2.37e-07 2.54e-07 2.56e-07 1.58941e-320 +gather: 1 (ozark): 2.3465e-05: breakdown: 1.414e-06 3.56e-07 3.11e-07 2.95e-07 2.71e-07 2.64e-07 2.56e-07 2.45e-07 2.34e-07 2.31e-07 2.29e-07 2.26e-07 2.29e-07 2.23e-07 2.29e-07 2.25e-07 2.23e-07 2.25e-07 2.85e-07 2.19e-07 2.19e-07 2.22e-07 2.87e-07 2.18e-07 2.2e-07 2.22e-07 2.19e-07 2.2e-07 2.29e-07 2.23e-07 2.24e-07 2.18e-07 2.25e-07 2.19e-07 2.21e-07 2.18e-07 2.23e-07 2.2e-07 2.17e-07 2.28e-07 2.19e-07 2.19e-07 2.2e-07 2.14e-07 2.16e-07 2.18e-07 2.14e-07 2.13e-07 2.13e-07 2.13e-07 2.15e-07 2.17e-07 2.13e-07 2.14e-07 2.16e-07 2.19e-07 2.15e-07 2.19e-07 2.15e-07 2.12e-07 2.15e-07 2.14e-07 2.2e-07 2.19e-07 2.2e-07 2.14e-07 2.13e-07 2.14e-07 2.17e-07 2.13e-07 2.11e-07 2.15e-07 2.15e-07 2.17e-07 2.19e-07 2.12e-07 2.13e-07 2.16e-07 2.14e-07 2.18e-07 2.82e-07 2.16e-07 2.21e-07 2.13e-07 2.13e-07 2.15e-07 2.13e-07 2.15e-07 2.17e-07 2.14e-07 2.15e-07 2.17e-07 2.12e-07 2.17e-07 2.13e-07 2.16e-07 2.19e-07 2.16e-07 2.16e-07 1.58941e-320 +gather: 2 (ozark): 2.35e-05: breakdown: 1.522e-06 3.73e-07 3.35e-07 2.75e-07 3.24e-07 3.07e-07 2.99e-07 2.41e-07 2.4e-07 2.4e-07 2.41e-07 2.18e-07 2.18e-07 2.19e-07 2.21e-07 2.18e-07 2.2e-07 2.11e-07 2.14e-07 2.17e-07 2.17e-07 2.2e-07 2.14e-07 2.18e-07 2.15e-07 2.18e-07 2.13e-07 2.15e-07 2.12e-07 2.12e-07 2.16e-07 2.15e-07 2.14e-07 2.21e-07 2.12e-07 2.14e-07 2.19e-07 2.12e-07 2.15e-07 2.15e-07 2.12e-07 2.14e-07 2.1e-07 2.12e-07 2.61e-07 2.14e-07 2.13e-07 2.37e-07 2.15e-07 2.2e-07 2.18e-07 2.18e-07 2.19e-07 2.2e-07 2.17e-07 2.2e-07 2.18e-07 2.18e-07 2.2e-07 2.17e-07 2.18e-07 2.16e-07 2.15e-07 2.13e-07 2.14e-07 2.18e-07 2.15e-07 2.15e-07 2.13e-07 2.11e-07 2.12e-07 2.1e-07 2.14e-07 2.13e-07 2.13e-07 2.16e-07 2.17e-07 2.12e-07 2.16e-07 2.13e-07 2.14e-07 2.13e-07 2.13e-07 2.14e-07 2.11e-07 2.13e-07 2.14e-07 2.11e-07 2.12e-07 2.16e-07 2.1e-07 3.21e-07 2.14e-07 2.14e-07 2.16e-07 2.1e-07 2.17e-07 2.13e-07 2.17e-07 1.58941e-320 +gather: 3 (ozark): 2.3176e-05: breakdown: 1.772e-06 3.98e-07 2.99e-07 3.59e-07 2.62e-07 2.66e-07 2.55e-07 2.35e-07 2.24e-07 2.32e-07 2.2e-07 2.16e-07 2.22e-07 2.19e-07 2.16e-07 2.16e-07 2.16e-07 2.09e-07 2.31e-07 2.15e-07 2.12e-07 2.15e-07 2.13e-07 2.14e-07 2.13e-07 2.11e-07 2.14e-07 2.11e-07 2.12e-07 2.11e-07 2.15e-07 2.1e-07 2.13e-07 2.03e-07 2.03e-07 2.14e-07 2.1e-07 2.14e-07 2.11e-07 2.11e-07 2.15e-07 2.16e-07 2.12e-07 2.15e-07 2.11e-07 2.09e-07 2.1e-07 2.12e-07 2.1e-07 2.12e-07 2.08e-07 2.72e-07 2.08e-07 2.08e-07 2.09e-07 2.07e-07 2.05e-07 2.09e-07 2.08e-07 2.06e-07 2.09e-07 2.09e-07 2.09e-07 2.11e-07 2.12e-07 2.1e-07 2.07e-07 2.08e-07 2.1e-07 2.08e-07 2.11e-07 2.1e-07 2.09e-07 2.11e-07 2.12e-07 2.11e-07 2.11e-07 2.13e-07 2.11e-07 2.1e-07 2.1e-07 2.11e-07 2.1e-07 2.1e-07 2.09e-07 2.08e-07 2.1e-07 2.12e-07 2.12e-07 2.1e-07 2.14e-07 2.08e-07 2.1e-07 2.12e-07 2.1e-07 2.12e-07 2.12e-07 2.1e-07 2.1e-07 1.58941e-320 diff --git a/tests/unit/detection/test_slow_node_detector.py b/tests/unit/detection/test_slow_node_detector.py index f0b25aa..545a07e 100644 --- a/tests/unit/detection/test_slow_node_detector.py +++ b/tests/unit/detection/test_slow_node_detector.py @@ -18,7 +18,7 @@ def setUp(self): self.spn = 1 # Determine expected values - self.expected_slow_ranks = [1, 2, 4, 6, 7, 9, 10, 11] + self.expected_slow_ranks = [1, 2] self.expected_slow_nodes = [] self.expected_overheated_nodes = ["ozark"] @@ -28,6 +28,8 @@ def setUp(self): sensors=self.sensors_file, num_nodes=2, pct=0.05, + benchmark="level3", + type="double", spn=self.spn, rpn=self.rpn, plot_rank_breakdowns=False,