From 52b1c2bd19a25554c706851f92ba6a06f41936ee Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 15 May 2022 13:08:22 +0200 Subject: [PATCH 01/21] Run phase cleanup only if phase is run. --- src/phase_ior_easy.c | 2 +- src/phase_ior_hard.c | 2 +- src/phase_ior_rnd_write1MB.c | 2 +- src/phase_ior_rnd_write4K.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/phase_ior_easy.c b/src/phase_ior_easy.c index ba352e3..ae5ef84 100644 --- a/src/phase_ior_easy.c +++ b/src/phase_ior_easy.c @@ -21,7 +21,7 @@ static void validate(void){ } static void cleanup(void){ - if (opt.dry_run) return; + if (opt.dry_run || ! ior_easy_o.run) return; if(opt.rank == 0){ char filename[PATH_MAX]; diff --git a/src/phase_ior_hard.c b/src/phase_ior_hard.c index 9162943..d052098 100644 --- a/src/phase_ior_hard.c +++ b/src/phase_ior_hard.c @@ -21,7 +21,7 @@ static void validate(void){ } static void cleanup(void){ - if (opt.dry_run) return; + if (opt.dry_run || ! ior_hard_o.run) return; if( opt.rank == 0){ char filename[PATH_MAX]; diff --git a/src/phase_ior_rnd_write1MB.c b/src/phase_ior_rnd_write1MB.c index e256f2a..a1a28d3 100644 --- a/src/phase_ior_rnd_write1MB.c +++ b/src/phase_ior_rnd_write1MB.c @@ -28,7 +28,7 @@ static void validate(void){ } static void cleanup(void){ - if (opt.dry_run) return; + if (opt.dry_run || ! ior_rnd1MB_o.run) return; if(opt.rank == 0){ char filename[PATH_MAX]; diff --git a/src/phase_ior_rnd_write4K.c b/src/phase_ior_rnd_write4K.c index d21d6b8..4baaa11 100644 --- a/src/phase_ior_rnd_write4K.c +++ b/src/phase_ior_rnd_write4K.c @@ -28,7 +28,7 @@ static void validate(void){ } static void cleanup(void){ - if (opt.dry_run) return; + if (opt.dry_run || ! ior_rnd4K_o.run) return; if(opt.rank == 0){ char filename[PATH_MAX]; From e90dcaef4e02e119bba45e6d39dfda5c060f074b Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 15 May 2022 14:01:10 +0200 Subject: [PATCH 02/21] Added hook for concurrent benchmark and allow to specify rank for output --- Makefile | 2 +- include/io500-phase.h | 5 ++++- include/io500-util.h | 2 +- src/main.c | 6 ++++-- src/phase-definitions.h | 2 ++ src/phase_ior_easy_read.c | 2 +- src/phase_ior_easy_write.c | 2 +- src/phase_ior_hard_read.c | 2 +- src/phase_ior_hard_write.c | 2 +- src/phase_ior_rnd_read1MB.c | 2 +- src/phase_ior_rnd_read4K.c | 2 +- src/phase_ior_rnd_write1MB.c | 2 +- src/phase_ior_rnd_write4K.c | 2 +- src/phase_mdtest_easy_delete.c | 2 +- src/phase_mdtest_easy_stat.c | 2 +- src/phase_mdtest_easy_write.c | 2 +- src/phase_mdtest_hard_delete.c | 2 +- src/phase_mdtest_hard_read.c | 2 +- src/phase_mdtest_hard_stat.c | 2 +- src/phase_mdtest_hard_write.c | 2 +- src/phase_mdworkbench_bench.c | 2 +- src/phase_mdworkbench_create.c | 2 +- src/phase_mdworkbench_delete.c | 2 +- src/util.c | 4 ++-- 24 files changed, 32 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index ee5cee9..3e72245 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ vpath %.h $(SEARCHPATH) DEPS += io500-util.h io500-debug.h io500-opt.h OBJSO += util.o OBJSO += ini-parse.o phase_dbg.o phase_opt.o phase_timestamp.o -OBJSO += phase_find.o phase_find_easy.o phase_find_hard.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd1MB.o phase_ior_rnd4K.o phase_ior_rnd_write4K.o phase_ior_rnd_read4K.o phase_ior_rnd_write1MB.o phase_ior_rnd_read1MB.o phase_mdworkbench.o phase_mdworkbench_create.o phase_mdworkbench_delete.o phase_mdworkbench_bench.o +OBJSO += phase_find.o phase_find_easy.o phase_find_hard.o phase_ior_easy.o phase_ior_easy_read.o phase_mdtest.o phase_ior.o phase_ior_easy_write.o phase_ior_hard.o phase_ior_hard_read.o phase_ior_hard_write.o phase_mdtest_easy.o phase_mdtest_easy_delete.o phase_mdtest_easy_stat.o phase_mdtest_easy_write.o phase_mdtest_hard.o phase_mdtest_hard_delete.o phase_mdtest_hard_read.o phase_mdtest_hard_stat.o phase_mdtest_hard_write.o phase_ior_rnd1MB.o phase_ior_rnd4K.o phase_ior_rnd_write4K.o phase_ior_rnd_read4K.o phase_ior_rnd_write1MB.o phase_ior_rnd_read1MB.o phase_mdworkbench.o phase_mdworkbench_create.o phase_mdworkbench_delete.o phase_mdworkbench_bench.o phase_concurrent.o OBJS = $(patsubst %,./build/%,$(OBJSO)) diff --git a/include/io500-phase.h b/include/io500-phase.h index b61a254..c97d2ea 100644 --- a/include/io500-phase.h +++ b/include/io500-phase.h @@ -11,6 +11,7 @@ typedef enum { IO500_NO_SCORE, IO500_SCORE_MD, IO500_SCORE_BW, + IO500_SCORE_CONCURRENT, IO500_SCORE_LAST } io500_phase_score_group; @@ -37,7 +38,7 @@ typedef struct{ io500_phase_score_group group; } u_phase_t; -#define IO500_PHASES (2 + 1 + 2*3 + 3 + 3 + 4 + 5 + 3 + 4) +#define IO500_PHASES (2 + 1 + 2*3 + 3 + 3 + 4 + 5 + 3 + 4 + 1) extern u_phase_t p_opt; extern u_phase_t p_debug; @@ -65,6 +66,8 @@ extern u_phase_t p_mdworkbench_create; extern u_phase_t p_mdworkbench_bench; extern u_phase_t p_mdworkbench_delete; +extern u_phase_t p_concurrent; + extern u_phase_t p_ior_easy; extern u_phase_t p_ior_easy_write; extern u_phase_t p_ior_easy_read; diff --git a/include/io500-util.h b/include/io500-util.h index 997cd14..23b786d 100644 --- a/include/io500-util.h +++ b/include/io500-util.h @@ -82,7 +82,7 @@ void u_print_timestamp(FILE * out); void * u_malloc(int size); -FILE * u_res_file_prep(char const * name); +FILE * u_res_file_prep(char const * name, int rank); void u_res_file_close(FILE * out); uint32_t u_phase_unique_random_number(char const * phase_name); diff --git a/src/main.c b/src/main.c index 5a2bdce..53b23f3 100644 --- a/src/main.c +++ b/src/main.c @@ -22,7 +22,9 @@ FILE* file_out = NULL; static char const * io500_phase_str[IO500_SCORE_LAST] = { "NO SCORE", "MD", - "BW"}; + "BW", + "CONCURRENT" + }; static void prepare_aiori(void){ // check selected API, might be followed by API options @@ -132,7 +134,7 @@ static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t int numbers = 0; p += sprintf(p, " %s = (", io500_phase_str[g]); for(int i=0; i < IO500_PHASES; i++){ - if(phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){ + if( phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){ double t = phases[i]->score; if(t <= 0){ continue; diff --git a/src/phase-definitions.h b/src/phase-definitions.h index 3a9b16d..4ae7c9c 100644 --- a/src/phase-definitions.h +++ b/src/phase-definitions.h @@ -36,6 +36,8 @@ static u_phase_t * phases[IO500_PHASES] = { & p_mdworkbench_bench, + & p_concurrent, + & p_ior_easy_read, & p_mdtest_easy_stat, diff --git a/src/phase_ior_easy_read.c b/src/phase_ior_easy_read.c index 095b455..3358d28 100644 --- a/src/phase_ior_easy_read.c +++ b/src/phase_ior_easy_read.c @@ -49,7 +49,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_ior_easy_read.name); + FILE * out = u_res_file_prep(p_ior_easy_read.name, opt.rank); return ior_process_read(argv, out, & o.res); } diff --git a/src/phase_ior_easy_write.c b/src/phase_ior_easy_write.c index 4cd347f..a8b6729 100644 --- a/src/phase_ior_easy_write.c +++ b/src/phase_ior_easy_write.c @@ -51,7 +51,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_ior_easy_write.name); + FILE * out = u_res_file_prep(p_ior_easy_write.name, opt.rank); return ior_process_write(argv, out, & o.res); } diff --git a/src/phase_ior_hard_read.c b/src/phase_ior_hard_read.c index 5b35ee7..d8212ee 100644 --- a/src/phase_ior_hard_read.c +++ b/src/phase_ior_hard_read.c @@ -51,7 +51,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_ior_hard_read.name); + FILE * out = u_res_file_prep(p_ior_hard_read.name, opt.rank); return ior_process_read(argv, out, & o.res); } diff --git a/src/phase_ior_hard_write.c b/src/phase_ior_hard_write.c index 91ea568..42f6776 100644 --- a/src/phase_ior_hard_write.c +++ b/src/phase_ior_hard_write.c @@ -52,7 +52,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_ior_hard_write.name); + FILE * out = u_res_file_prep(p_ior_hard_write.name, opt.rank); return ior_process_write(argv, out, & o.res); } diff --git a/src/phase_ior_rnd_read1MB.c b/src/phase_ior_rnd_read1MB.c index 0693d41..c8c7b9b 100644 --- a/src/phase_ior_rnd_read1MB.c +++ b/src/phase_ior_rnd_read1MB.c @@ -44,7 +44,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_ior_rnd1MB_read.name); + FILE * out = u_res_file_prep(p_ior_rnd1MB_read.name, opt.rank); return ior_process_read(argv, out, & o.res); } diff --git a/src/phase_ior_rnd_read4K.c b/src/phase_ior_rnd_read4K.c index 275b88c..e66520a 100644 --- a/src/phase_ior_rnd_read4K.c +++ b/src/phase_ior_rnd_read4K.c @@ -44,7 +44,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_ior_rnd4K_read.name); + FILE * out = u_res_file_prep(p_ior_rnd4K_read.name, opt.rank); return ior_process_read(argv, out, & o.res); } diff --git a/src/phase_ior_rnd_write1MB.c b/src/phase_ior_rnd_write1MB.c index a1a28d3..01b6250 100644 --- a/src/phase_ior_rnd_write1MB.c +++ b/src/phase_ior_rnd_write1MB.c @@ -62,7 +62,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_ior_rnd1MB_write.name); + FILE * out = u_res_file_prep(p_ior_rnd1MB_write.name, opt.rank); return ior_process_write(argv, out, & o.res); } diff --git a/src/phase_ior_rnd_write4K.c b/src/phase_ior_rnd_write4K.c index 4baaa11..bda7b27 100644 --- a/src/phase_ior_rnd_write4K.c +++ b/src/phase_ior_rnd_write4K.c @@ -62,7 +62,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_ior_rnd4K_write.name); + FILE * out = u_res_file_prep(p_ior_rnd4K_write.name, opt.rank); return ior_process_write(argv, out, & o.res); } diff --git a/src/phase_mdtest_easy_delete.c b/src/phase_mdtest_easy_delete.c index 7504750..92b85bf 100644 --- a/src/phase_mdtest_easy_delete.c +++ b/src/phase_mdtest_easy_delete.c @@ -35,7 +35,7 @@ static double run(void){ return 0; } - FILE * out = u_res_file_prep(p_mdtest_easy_delete.name); + FILE * out = u_res_file_prep(p_mdtest_easy_delete.name, opt.rank); p_mdtest_run(argv, out, & o.res, MDTEST_FILE_REMOVE_NUM); return o.res.rate; diff --git a/src/phase_mdtest_easy_stat.c b/src/phase_mdtest_easy_stat.c index 5fa2c81..db39181 100644 --- a/src/phase_mdtest_easy_stat.c +++ b/src/phase_mdtest_easy_stat.c @@ -35,7 +35,7 @@ static double run(void){ return 0; } - FILE * out = u_res_file_prep(p_mdtest_easy_stat.name); + FILE * out = u_res_file_prep(p_mdtest_easy_stat.name, opt.rank); p_mdtest_run(argv, out, & o.res, MDTEST_FILE_STAT_NUM); return o.res.rate; } diff --git a/src/phase_mdtest_easy_write.c b/src/phase_mdtest_easy_write.c index f0f2b65..f1db4a3 100644 --- a/src/phase_mdtest_easy_write.c +++ b/src/phase_mdtest_easy_write.c @@ -42,7 +42,7 @@ static double run(void){ return 0; } - FILE * out = u_res_file_prep(p_mdtest_easy_write.name); + FILE * out = u_res_file_prep(p_mdtest_easy_write.name, opt.rank); p_mdtest_run(argv, out, & o.res, MDTEST_FILE_CREATE_NUM); PRINT_PAIR("rate-stonewall", "%f\n", o.res.rate_stonewall); diff --git a/src/phase_mdtest_hard_delete.c b/src/phase_mdtest_hard_delete.c index 2a53ca7..9699569 100644 --- a/src/phase_mdtest_hard_delete.c +++ b/src/phase_mdtest_hard_delete.c @@ -35,7 +35,7 @@ static double run(void){ return 0; } - FILE * out = u_res_file_prep(p_mdtest_hard_delete.name); + FILE * out = u_res_file_prep(p_mdtest_hard_delete.name, opt.rank); p_mdtest_run(argv, out, & o.res, MDTEST_FILE_REMOVE_NUM); return o.res.rate; diff --git a/src/phase_mdtest_hard_read.c b/src/phase_mdtest_hard_read.c index 6152ca5..86238af 100644 --- a/src/phase_mdtest_hard_read.c +++ b/src/phase_mdtest_hard_read.c @@ -36,7 +36,7 @@ static double run(void){ return 0; } - FILE * out = u_res_file_prep(p_mdtest_hard_read.name); + FILE * out = u_res_file_prep(p_mdtest_hard_read.name, opt.rank); p_mdtest_run(argv, out, & o.res, MDTEST_FILE_READ_NUM); return o.res.rate; diff --git a/src/phase_mdtest_hard_stat.c b/src/phase_mdtest_hard_stat.c index cb3f7ec..2b48944 100644 --- a/src/phase_mdtest_hard_stat.c +++ b/src/phase_mdtest_hard_stat.c @@ -35,7 +35,7 @@ static double run(void){ return 0; } - FILE * out = u_res_file_prep(p_mdtest_hard_stat.name); + FILE * out = u_res_file_prep(p_mdtest_hard_stat.name, opt.rank); p_mdtest_run(argv, out, & o.res, MDTEST_FILE_STAT_NUM); return o.res.rate; } diff --git a/src/phase_mdtest_hard_write.c b/src/phase_mdtest_hard_write.c index fce389b..590ea6e 100644 --- a/src/phase_mdtest_hard_write.c +++ b/src/phase_mdtest_hard_write.c @@ -45,7 +45,7 @@ static double run(void){ return 0; } - FILE * out = u_res_file_prep(p_mdtest_hard_write.name); + FILE * out = u_res_file_prep(p_mdtest_hard_write.name, opt.rank); p_mdtest_run(argv, out, & o.res, MDTEST_FILE_CREATE_NUM); PRINT_PAIR("rate-stonewall", "%f\n", o.res.rate_stonewall); diff --git a/src/phase_mdworkbench_bench.c b/src/phase_mdworkbench_bench.c index 9e7010a..2fa3026 100644 --- a/src/phase_mdworkbench_bench.c +++ b/src/phase_mdworkbench_bench.c @@ -33,7 +33,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_mdworkbench_bench.name); + FILE * out = u_res_file_prep(p_mdworkbench_bench.name, opt.rank); mdworkbench_process(argv, out, & o.res); if(o.res->count != 2){ INVALID("During the md-workbench phase not two iterations are performed but %d This invalidates your run.\n", o.res->count); diff --git a/src/phase_mdworkbench_create.c b/src/phase_mdworkbench_create.c index eff9041..bfa55c1 100644 --- a/src/phase_mdworkbench_create.c +++ b/src/phase_mdworkbench_create.c @@ -31,7 +31,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_mdworkbench_create.name); + FILE * out = u_res_file_prep(p_mdworkbench_create.name, opt.rank); mdworkbench_process(argv, out, & o.res); PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); diff --git a/src/phase_mdworkbench_delete.c b/src/phase_mdworkbench_delete.c index af79b4f..0779fb3 100644 --- a/src/phase_mdworkbench_delete.c +++ b/src/phase_mdworkbench_delete.c @@ -30,7 +30,7 @@ static double run(void){ u_argv_free(argv); return 0; } - FILE * out = u_res_file_prep(p_mdworkbench_delete.name); + FILE * out = u_res_file_prep(p_mdworkbench_delete.name, opt.rank); mdworkbench_process(argv, out, & o.res); PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); diff --git a/src/util.c b/src/util.c index ff538e9..d022b75 100644 --- a/src/util.c +++ b/src/util.c @@ -236,9 +236,9 @@ void u_print_timestamp(FILE * out){ fprintf(out, "%s", buffer); } -FILE * u_res_file_prep(char const * name){ +FILE * u_res_file_prep(char const * name, int rank){ FILE * out = stdout; - if(opt.rank == 0){ + if(rank == 0){ char fname[2048]; sprintf(fname, "%s/%s.txt", opt.resdir, name); INFO_PAIR("result-file", "%s\n", fname); From 1099986bf99c3373c38e03e01394dae1fecc5bb8 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 15 May 2022 14:19:45 +0200 Subject: [PATCH 03/21] Initial version with some scoring --- src/phase_concurrent.c | 150 +++++++++++++++++++++++++++++++++++++++++ src/phase_concurrent.h | 7 ++ 2 files changed, 157 insertions(+) create mode 100644 src/phase_concurrent.c create mode 100644 src/phase_concurrent.h diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c new file mode 100644 index 0000000..9ffe68a --- /dev/null +++ b/src/phase_concurrent.c @@ -0,0 +1,150 @@ +#include +#include +#include + +#include +#include + +#define CBENCHS 3 + +typedef struct{ + int run; + char * command[CBENCHS]; + mdworkbench_results_t * mdw; + IOR_point_t * ior; +} opt_concurrent_bench; + +static opt_concurrent_bench o; + +static ini_option_t option[] = { + {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, + //{"", "XX", 0, INI_UINT, NULL, & o.XX}, + {NULL} }; + + +static double run(void){ + opt_concurrent_bench d = o; + + if(opt.mpi_size < 5){ + INVALID("The concurrent phase needs at least 5 processes\n"); + return 0; + } + + MPI_Comm ccom; + /* + Split processes into 3 groups and benchmarks: + benchmark 0 - 20% parallel write - ior easy + benchmark 1 - 40% parallel read 1 MB - rnd1MB read + benchmark 2 - 40% md-workbench for concurrent usage + */ + int workloads[] = {opt.mpi_size * 0.2, opt.mpi_size * 0.6, opt.mpi_size}; + int color = (opt.rank < workloads[0]) ? 0 : (opt.rank < workloads[1] ? 1 : 2); + UMPI_CHECK(MPI_Comm_split(MPI_COMM_WORLD, color, opt.rank, & ccom)); + + int crank; + MPI_Comm_rank(ccom, & crank); + /* printf("%d - %d - rank %d\n", opt.rank, color, crank); */ + PRINT_PAIR("benchmark", "%d\n", color); + + u_argv_t * argv[CBENCHS]; + for(int i=0; i < CBENCHS; i++){ + argv[i] = u_argv_create(); + } + + /* prepare benchmark settings */ + { + opt_ior_easy d = ior_easy_o; + ior_easy_add_params(argv[0]); + u_argv_push(argv[0], "-w"); /* write file */ + u_argv_push(argv[0], "-D"); /* deadline for stonewall in seconds */ + u_argv_push_printf(argv[0], "%d", opt.stonewall); + //u_argv_push_default_if_set_api_options(argv[0], "-a", d.api, o.api); + u_argv_push_default_if_set_api_options(argv[0], "-a", d.api, d.api); // TODO USE RND WRITE OPTION + u_argv_push(argv[0], "-O"); + u_argv_push_printf(argv[0], "saveRankPerformanceDetailsCSV=%s/concurrent-ior-easy-write.csv", opt.resdir); + o.command[0] = u_flatten_argv(argv[0]); + PRINT_PAIR("exe-easy-write", "%s\n", o.command[0]); + } + + { + opt_ior_rnd d = ior_rnd1MB_o; + ior_rnd1MB_add_params(argv[1]); + u_argv_push(argv[1], "-r"); + u_argv_push(argv[1], "-R"); + u_argv_push_default_if_set_api_options(argv[1], "-a", d.api, d.api); // TODO USE RND WRITE OPTION + u_argv_push(argv[1], "-O"); + u_argv_push_printf(argv[1], "saveRankPerformanceDetailsCSV=%s/concurrent-ior-rnd1MB-read.csv", opt.resdir); + u_argv_push(argv[0], "-D"); /* deadline for stonewall in seconds */ + u_argv_push_printf(argv[0], "%d", opt.stonewall); + o.command[1] = u_flatten_argv(argv[1]); + PRINT_PAIR("exe-rnd1MB-read", "%s\n", o.command[1]); + } + + { + mdworkbench_add_params(argv[2], 0); + u_argv_push(argv[2], "-2"); /* run benchmark phase */ + u_argv_push(argv[2], "-R=1"); /* 1 repetition */ + u_argv_push(argv[2], "-X"); /* turn read verification on */ + u_argv_push_printf(argv[2], "-w=%d", opt.stonewall); + o.command[2] = u_flatten_argv(argv[2]); + PRINT_PAIR("exe-md-workbench", "%s\n", o.command[2]); + } + + if(opt.dry_run || d.run == 0){ + for(int i=0; i < CBENCHS; i++){ + u_argv_free(argv[i]); + } + return 0; + } + double score = 0; + if(0){ + if(color == 0){ + // run ior easy write + FILE * out = u_res_file_prep("concurrent-ior-easy-write", crank); + score = ior_process_write(argv[0], out, & o.ior); + }else if(color == 1){ + // run ior rnd 1MB read + FILE * out = u_res_file_prep("concurrent-rnd1MB-read", crank); + score = ior_process_read(argv[1], out, & o.ior); + }else if(color == 2){ + // run md_workbench + FILE * out = u_res_file_prep("concurrent-md-workbench", crank); + mdworkbench_process(argv[2], out, & o.mdw); + /* calculate score */ + double rate = o.mdw->result[0].rate; + PRINT_PAIR("maxOpTime", "%f\n", o.mdw->result[0].max_op_time); + score = rate / 1000.0; + } + } + score = color + 1; + + if(crank == 0){ + // exchange scores + double scores[CBENCHS]; + if(opt.rank == 0){ + double aggregated_score = score * workloads[0]; + //printf("%f - %f - count: %d\n", score, aggregated_score, workloads[0]); + for(int i=1; i < CBENCHS; i++){ + UMPI_CHECK(MPI_Recv(& scores[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); + aggregated_score += scores[i] * (workloads[i] - workloads[i-1]); + //printf("%f - %f - count: %d\n", scores[i], aggregated_score, (workloads[i] - workloads[i-1])); + } + score = aggregated_score / opt.mpi_size; + //printf("average: %f\n", score); + }else{ + UMPI_CHECK(MPI_Send(& score, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD)); + } + } + return score; +} + + +u_phase_t p_concurrent = { + "concurrent", + IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL, + option, + NULL, + run, + 0, + .group = IO500_SCORE_CONCURRENT, +}; diff --git a/src/phase_concurrent.h b/src/phase_concurrent.h new file mode 100644 index 0000000..6f253b7 --- /dev/null +++ b/src/phase_concurrent.h @@ -0,0 +1,7 @@ +#ifndef IO500_PHASE_CONCURRENT_H +#define IO500_PHASE_CONCURRENT_H + +#include +#include + +#endif From d752cb0de8403df7e10c340daf9d61cc758b9831 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 15 May 2022 15:30:29 +0200 Subject: [PATCH 04/21] First version of concurrent I/O. This version is for testing. It executes benchmark 0 - 20% procs - parallel write - ior easy benchmark 1 - 40% procs - parallel read 1 MB - rnd1MB read benchmark 2 - 40% procs - md-workbench for concurrent usage It does provoke errors in the md-workbench cleanup phase and extra output. These are not score relevant, though. --- src/phase_concurrent.c | 34 +++++++++++++++++++++------------- src/phase_ior.c | 8 ++++---- src/phase_ior.h | 6 +++--- src/phase_ior_easy.c | 8 +++++--- src/phase_ior_easy_read.c | 4 ++-- src/phase_ior_easy_write.c | 4 ++-- src/phase_ior_hard_read.c | 2 +- src/phase_ior_hard_write.c | 2 +- src/phase_ior_rnd_read1MB.c | 2 +- src/phase_ior_rnd_read4K.c | 2 +- src/phase_ior_rnd_write1MB.c | 2 +- src/phase_ior_rnd_write4K.c | 2 +- src/phase_mdworkbench.c | 4 ++-- src/phase_mdworkbench.h | 2 +- src/phase_mdworkbench_bench.c | 2 +- src/phase_mdworkbench_create.c | 2 +- src/phase_mdworkbench_delete.c | 2 +- 17 files changed, 49 insertions(+), 39 deletions(-) diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index 9ffe68a..fc9c230 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -12,6 +12,7 @@ typedef struct{ char * command[CBENCHS]; mdworkbench_results_t * mdw; IOR_point_t * ior; + int color; // benchmark that is run } opt_concurrent_bench; static opt_concurrent_bench o; @@ -21,7 +22,6 @@ static ini_option_t option[] = { //{"", "XX", 0, INI_UINT, NULL, & o.XX}, {NULL} }; - static double run(void){ opt_concurrent_bench d = o; @@ -39,6 +39,7 @@ static double run(void){ */ int workloads[] = {opt.mpi_size * 0.2, opt.mpi_size * 0.6, opt.mpi_size}; int color = (opt.rank < workloads[0]) ? 0 : (opt.rank < workloads[1] ? 1 : 2); + o.color = color; UMPI_CHECK(MPI_Comm_split(MPI_COMM_WORLD, color, opt.rank, & ccom)); int crank; @@ -54,7 +55,7 @@ static double run(void){ /* prepare benchmark settings */ { opt_ior_easy d = ior_easy_o; - ior_easy_add_params(argv[0]); + ior_easy_add_params(argv[0], 0); u_argv_push(argv[0], "-w"); /* write file */ u_argv_push(argv[0], "-D"); /* deadline for stonewall in seconds */ u_argv_push_printf(argv[0], "%d", opt.stonewall); @@ -82,10 +83,12 @@ static double run(void){ { mdworkbench_add_params(argv[2], 0); - u_argv_push(argv[2], "-2"); /* run benchmark phase */ u_argv_push(argv[2], "-R=1"); /* 1 repetition */ u_argv_push(argv[2], "-X"); /* turn read verification on */ u_argv_push_printf(argv[2], "-w=%d", opt.stonewall); + u_argv_push_printf(argv, "-o=%s/concurrent-mdworkbench", opt.datadir); + u_argv_push_printf(argv, "--run-info-file=%s/concurrent-mdworkbench.status", opt.resdir ); + u_argv_push(argv[2], "-2"); /* run pre-create or benchmarking phase */ o.command[2] = u_flatten_argv(argv[2]); PRINT_PAIR("exe-md-workbench", "%s\n", o.command[2]); } @@ -95,40 +98,45 @@ static double run(void){ u_argv_free(argv[i]); } return 0; - } + } double score = 0; - if(0){ - if(color == 0){ + if(color == 0){ // run ior easy write + if(ior_easy_o.filePerProc){ + char filename[2048]; + sprintf(filename, "ior-easy/ior_file_easy.%08d", opt.rank); + u_purge_file(filename); + } FILE * out = u_res_file_prep("concurrent-ior-easy-write", crank); - score = ior_process_write(argv[0], out, & o.ior); + score = ior_process_write(argv[0], out, & o.ior, ccom); }else if(color == 1){ // run ior rnd 1MB read FILE * out = u_res_file_prep("concurrent-rnd1MB-read", crank); - score = ior_process_read(argv[1], out, & o.ior); + score = ior_process_read(argv[1], out, & o.ior, ccom); }else if(color == 2){ // run md_workbench FILE * out = u_res_file_prep("concurrent-md-workbench", crank); - mdworkbench_process(argv[2], out, & o.mdw); + mdworkbench_process(argv[2], out, & o.mdw, ccom); /* calculate score */ double rate = o.mdw->result[0].rate; PRINT_PAIR("maxOpTime", "%f\n", o.mdw->result[0].max_op_time); score = rate / 1000.0; } - } - score = color + 1; if(crank == 0){ - // exchange scores + // exchange scores, calculate arithmetic mean score double scores[CBENCHS]; if(opt.rank == 0){ double aggregated_score = score * workloads[0]; - //printf("%f - %f - count: %d\n", score, aggregated_score, workloads[0]); + //printf("%f - %f - count: %d\n", score, aggregated_score, workloads[0]); for(int i=1; i < CBENCHS; i++){ UMPI_CHECK(MPI_Recv(& scores[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); aggregated_score += scores[i] * (workloads[i] - workloads[i-1]); //printf("%f - %f - count: %d\n", scores[i], aggregated_score, (workloads[i] - workloads[i-1])); } + PRINT_PAIR("score-ior-easy-write", "%f\n", scores[0]); + PRINT_PAIR("score-ior-rnd1MB-read", "%f\n", scores[1]); + PRINT_PAIR("score-ior-md-workbench", "%f\n", scores[2]); score = aggregated_score / opt.mpi_size; //printf("average: %f\n", score); }else{ diff --git a/src/phase_ior.c b/src/phase_ior.c index 2a0ed18..494eacb 100644 --- a/src/phase_ior.c +++ b/src/phase_ior.c @@ -2,8 +2,8 @@ #include -double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){ - IOR_test_t * test = ior_run(argv->size, argv->vector, MPI_COMM_WORLD, out); +double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com){ + IOR_test_t * test = ior_run(argv->size, argv->vector, com, out); assert(test); IOR_results_t * res = test->results; assert(res); @@ -31,8 +31,8 @@ double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){ return tp; } -double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out){ - IOR_results_t * res = ior_run(argv->size, argv->vector, MPI_COMM_WORLD, out)->results; +double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com){ + IOR_results_t * res = ior_run(argv->size, argv->vector, com, out)->results; u_res_file_close(out); u_argv_free(argv); diff --git a/src/phase_ior.h b/src/phase_ior.h index 63d3995..aa4b4b0 100644 --- a/src/phase_ior.h +++ b/src/phase_ior.h @@ -41,13 +41,13 @@ extern opt_ior_rnd ior_rnd4K_o; extern opt_ior_rnd ior_rnd1MB_o; -void ior_easy_add_params(u_argv_t * argv); +void ior_easy_add_params(u_argv_t * argv, int useStatusFile); void ior_hard_add_params(u_argv_t * argv); void ior_rnd4K_add_params(u_argv_t * argv); void ior_rnd1MB_add_params(u_argv_t * argv); // Generic helpers -double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out); -double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out); +double ior_process_write(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com); +double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI_Comm com); #endif diff --git a/src/phase_ior_easy.c b/src/phase_ior_easy.c index ae5ef84..82fde10 100644 --- a/src/phase_ior_easy.c +++ b/src/phase_ior_easy.c @@ -42,7 +42,7 @@ static void cleanup(void){ } } -void ior_easy_add_params(u_argv_t * argv){ +void ior_easy_add_params(u_argv_t * argv, int useStatusFile){ opt_ior_easy d = ior_easy_o; u_argv_push(argv, "./ior"); @@ -66,8 +66,10 @@ void ior_easy_add_params(u_argv_t * argv){ u_argv_push(argv, "-e"); /* fsync upon write close */ u_argv_push(argv, "-o"); /* filename for output file */ u_argv_push_printf(argv, "%s/ior-easy/ior_file_easy", opt.datadir); - u_argv_push(argv, "-O"); /* additional IOR options */ - u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-easy.stonewall", opt.resdir); + if(useStatusFile){ + u_argv_push(argv, "-O"); /* additional IOR options */ + u_argv_push_printf(argv, "stoneWallingStatusFile=%s/ior-easy.stonewall", opt.resdir); + } u_argv_push(argv, "-t"); /* transfer size */ u_argv_push(argv, d.transferSize); u_argv_push(argv, "-b"); /* blocksize in bytes */ diff --git a/src/phase_ior_easy_read.c b/src/phase_ior_easy_read.c index 3358d28..59eddab 100644 --- a/src/phase_ior_easy_read.c +++ b/src/phase_ior_easy_read.c @@ -30,7 +30,7 @@ static double run(void){ opt_ior_easy d = ior_easy_o; u_argv_t * argv = u_argv_create(); - ior_easy_add_params(argv); + ior_easy_add_params(argv, 1); u_argv_push(argv, "-r"); /* read existing file */ u_argv_push(argv, "-R"); /* verify data read */ u_argv_push_default_if_set_api_options(argv, "-a", d.api, o.api); @@ -50,7 +50,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_ior_easy_read.name, opt.rank); - return ior_process_read(argv, out, & o.res); + return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD); } diff --git a/src/phase_ior_easy_write.c b/src/phase_ior_easy_write.c index a8b6729..5d16b2b 100644 --- a/src/phase_ior_easy_write.c +++ b/src/phase_ior_easy_write.c @@ -29,7 +29,7 @@ static double run(void){ opt_ior_easy d = ior_easy_o; u_argv_t * argv = u_argv_create(); - ior_easy_add_params(argv); + ior_easy_add_params(argv, 1); u_argv_push(argv, "-w"); /* write file */ u_argv_push(argv, "-D"); /* deadline for stonewall in seconds */ u_argv_push_printf(argv, "%d", opt.stonewall); @@ -52,7 +52,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_ior_easy_write.name, opt.rank); - return ior_process_write(argv, out, & o.res); + return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD); } u_phase_t p_ior_easy_write = { diff --git a/src/phase_ior_hard_read.c b/src/phase_ior_hard_read.c index d8212ee..8c59650 100644 --- a/src/phase_ior_hard_read.c +++ b/src/phase_ior_hard_read.c @@ -52,7 +52,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_ior_hard_read.name, opt.rank); - return ior_process_read(argv, out, & o.res); + return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD); } diff --git a/src/phase_ior_hard_write.c b/src/phase_ior_hard_write.c index 42f6776..660c2d4 100644 --- a/src/phase_ior_hard_write.c +++ b/src/phase_ior_hard_write.c @@ -53,7 +53,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_ior_hard_write.name, opt.rank); - return ior_process_write(argv, out, & o.res); + return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD); } u_phase_t p_ior_hard_write = { diff --git a/src/phase_ior_rnd_read1MB.c b/src/phase_ior_rnd_read1MB.c index c8c7b9b..708da4b 100644 --- a/src/phase_ior_rnd_read1MB.c +++ b/src/phase_ior_rnd_read1MB.c @@ -45,7 +45,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_ior_rnd1MB_read.name, opt.rank); - return ior_process_read(argv, out, & o.res); + return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD); } diff --git a/src/phase_ior_rnd_read4K.c b/src/phase_ior_rnd_read4K.c index e66520a..8bd77ec 100644 --- a/src/phase_ior_rnd_read4K.c +++ b/src/phase_ior_rnd_read4K.c @@ -45,7 +45,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_ior_rnd4K_read.name, opt.rank); - return ior_process_read(argv, out, & o.res); + return ior_process_read(argv, out, & o.res, MPI_COMM_WORLD); } diff --git a/src/phase_ior_rnd_write1MB.c b/src/phase_ior_rnd_write1MB.c index 01b6250..3272dd7 100644 --- a/src/phase_ior_rnd_write1MB.c +++ b/src/phase_ior_rnd_write1MB.c @@ -63,7 +63,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_ior_rnd1MB_write.name, opt.rank); - return ior_process_write(argv, out, & o.res); + return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD); } u_phase_t p_ior_rnd1MB_write = { diff --git a/src/phase_ior_rnd_write4K.c b/src/phase_ior_rnd_write4K.c index bda7b27..627dc49 100644 --- a/src/phase_ior_rnd_write4K.c +++ b/src/phase_ior_rnd_write4K.c @@ -63,7 +63,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_ior_rnd4K_write.name, opt.rank); - return ior_process_write(argv, out, & o.res); + return ior_process_write(argv, out, & o.res, MPI_COMM_WORLD); } u_phase_t p_ior_rnd4K_write = { diff --git a/src/phase_mdworkbench.c b/src/phase_mdworkbench.c index a11cd6d..3db8577 100644 --- a/src/phase_mdworkbench.c +++ b/src/phase_mdworkbench.c @@ -32,8 +32,8 @@ static void validate(void){ } -void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out){ - mdworkbench_results_t * res = md_workbench_run(argv->size, argv->vector, MPI_COMM_WORLD, out); +void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out, MPI_Comm com){ + mdworkbench_results_t * res = md_workbench_run(argv->size, argv->vector, com, out); u_res_file_close(out); u_argv_free(argv); diff --git a/src/phase_mdworkbench.h b/src/phase_mdworkbench.h index 5ff0550..867b03c 100644 --- a/src/phase_mdworkbench.h +++ b/src/phase_mdworkbench.h @@ -16,6 +16,6 @@ typedef struct{ extern opt_mdworkbench mdworkbench_o; void mdworkbench_add_params(u_argv_t * argv, int is_create); -void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out); +void mdworkbench_process(u_argv_t * argv, FILE * out, mdworkbench_results_t ** res_out, MPI_Comm com); #endif diff --git a/src/phase_mdworkbench_bench.c b/src/phase_mdworkbench_bench.c index 2fa3026..bd68cd9 100644 --- a/src/phase_mdworkbench_bench.c +++ b/src/phase_mdworkbench_bench.c @@ -34,7 +34,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_mdworkbench_bench.name, opt.rank); - mdworkbench_process(argv, out, & o.res); + mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD); if(o.res->count != 2){ INVALID("During the md-workbench phase not two iterations are performed but %d This invalidates your run.\n", o.res->count); return 0.0; diff --git a/src/phase_mdworkbench_create.c b/src/phase_mdworkbench_create.c index bfa55c1..0805154 100644 --- a/src/phase_mdworkbench_create.c +++ b/src/phase_mdworkbench_create.c @@ -32,7 +32,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_mdworkbench_create.name, opt.rank); - mdworkbench_process(argv, out, & o.res); + mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD); PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); double rate = o.res->result[0].rate; diff --git a/src/phase_mdworkbench_delete.c b/src/phase_mdworkbench_delete.c index 0779fb3..2ba073e 100644 --- a/src/phase_mdworkbench_delete.c +++ b/src/phase_mdworkbench_delete.c @@ -31,7 +31,7 @@ static double run(void){ return 0; } FILE * out = u_res_file_prep(p_mdworkbench_delete.name, opt.rank); - mdworkbench_process(argv, out, & o.res); + mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD); PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); double rate = o.res->result[0].rate; From bee7d4ccb1a7e4ac9873d76784100424c287f039 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 15 May 2022 16:15:32 +0200 Subject: [PATCH 05/21] Bugfix: scoring to geo mean, only include concurrent phase in extended mode. --- include/io500-phase.h | 2 +- src/main.c | 10 +++++++--- src/phase_concurrent.c | 13 +++++++------ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/include/io500-phase.h b/include/io500-phase.h index c97d2ea..5c6f387 100644 --- a/include/io500-phase.h +++ b/include/io500-phase.h @@ -8,7 +8,7 @@ #define DEBUG_OPTION "INVALIDATES RUN; FOR DEBUGGING." typedef enum { - IO500_NO_SCORE, + IO500_NO_SCORE = 0, IO500_SCORE_MD, IO500_SCORE_BW, IO500_SCORE_CONCURRENT, diff --git a/src/main.c b/src/main.c index 53b23f3..d30290e 100644 --- a/src/main.c +++ b/src/main.c @@ -127,12 +127,16 @@ static void print_cfg_hash(FILE * out, ini_section_t ** cfg){ static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t * hash){ double overall_score = 1; - for(io500_phase_score_group g=1; g < IO500_SCORE_LAST; g++){ + int groups = IO500_SCORE_BW; + if(extended){ + groups = IO500_SCORE_CONCURRENT; + } + for(io500_phase_score_group g=1; g <= groups; g++){ char score_string[2048]; char *p = score_string; double score = 1; int numbers = 0; - p += sprintf(p, " %s = (", io500_phase_str[g]); + p += sprintf(p, " %s = (1.0", io500_phase_str[g]); for(int i=0; i < IO500_PHASES; i++){ if( phases[i]->group == g && (extended || ! (phases[i]->type & IO500_PHASE_FLAG_OPTIONAL)) ){ double t = phases[i]->score; @@ -157,7 +161,7 @@ static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t overall_score *= score; } - return sqrt(overall_score); + return pow(overall_score, 1.0 / groups); } static const char * io500_mode_str(io500_mode mode){ diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index fc9c230..37f4a05 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -86,8 +87,8 @@ static double run(void){ u_argv_push(argv[2], "-R=1"); /* 1 repetition */ u_argv_push(argv[2], "-X"); /* turn read verification on */ u_argv_push_printf(argv[2], "-w=%d", opt.stonewall); - u_argv_push_printf(argv, "-o=%s/concurrent-mdworkbench", opt.datadir); - u_argv_push_printf(argv, "--run-info-file=%s/concurrent-mdworkbench.status", opt.resdir ); + u_argv_push_printf(argv[2], "-o=%s/mdworkbench", opt.datadir); + u_argv_push_printf(argv[2], "--run-info-file=%s/mdworkbench.status", opt.resdir ); u_argv_push(argv[2], "-2"); /* run pre-create or benchmarking phase */ o.command[2] = u_flatten_argv(argv[2]); PRINT_PAIR("exe-md-workbench", "%s\n", o.command[2]); @@ -124,21 +125,21 @@ static double run(void){ } if(crank == 0){ - // exchange scores, calculate arithmetic mean score + // exchange scores, calculate geometric mean score double scores[CBENCHS]; if(opt.rank == 0){ + scores[0] = score; double aggregated_score = score * workloads[0]; //printf("%f - %f - count: %d\n", score, aggregated_score, workloads[0]); for(int i=1; i < CBENCHS; i++){ UMPI_CHECK(MPI_Recv(& scores[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); - aggregated_score += scores[i] * (workloads[i] - workloads[i-1]); + aggregated_score = aggregated_score * (scores[i] * (workloads[i] - workloads[i-1])); //printf("%f - %f - count: %d\n", scores[i], aggregated_score, (workloads[i] - workloads[i-1])); } PRINT_PAIR("score-ior-easy-write", "%f\n", scores[0]); PRINT_PAIR("score-ior-rnd1MB-read", "%f\n", scores[1]); PRINT_PAIR("score-ior-md-workbench", "%f\n", scores[2]); - score = aggregated_score / opt.mpi_size; - //printf("average: %f\n", score); + score = pow(aggregated_score / opt.mpi_size, 1.0/CBENCHS); }else{ UMPI_CHECK(MPI_Send(& score, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD)); } From fb1b3608bc686c7df2c2c99f6e49a128e0134238 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 21 May 2022 10:01:12 +0200 Subject: [PATCH 06/21] Allow parameterization of segment count for IOR RND tests. --- src/phase_ior.h | 1 + src/phase_ior_rnd1MB.c | 3 ++- src/phase_ior_rnd4K.c | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/phase_ior.h b/src/phase_ior.h index aa4b4b0..c4f56b0 100644 --- a/src/phase_ior.h +++ b/src/phase_ior.h @@ -34,6 +34,7 @@ typedef struct{ int random_prefill_bytes; uint64_t block_size; + int segments; int verbosity; } opt_ior_rnd; diff --git a/src/phase_ior_rnd1MB.c b/src/phase_ior_rnd1MB.c index 956c803..a2fa1aa 100644 --- a/src/phase_ior_rnd1MB.c +++ b/src/phase_ior_rnd1MB.c @@ -12,6 +12,7 @@ static ini_option_t option[] = { {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_rnd1MB_o.run}, {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_rnd1MB_o.verbosity}, {"randomPrefill", "Prefill the file with this blocksize in bytes, e.g., 2097152", 0, INI_INT, "0", & ior_rnd1MB_o.random_prefill_bytes}, + {"segmentCount", "Number of segments", 0, INI_INT, "10000000", & ior_rnd1MB_o.segments}, {NULL} }; @@ -55,7 +56,7 @@ void ior_rnd1MB_add_params(u_argv_t * argv){ u_argv_push(argv, "-k"); u_argv_push(argv, "-t=1048576"); u_argv_push_printf(argv, "-b=%ld", d.block_size); - u_argv_push_printf(argv, "-s=%d", 10000000); + u_argv_push_printf(argv, "-s=%d", d.segments); /* number of segments */ } u_phase_t p_ior_rnd1MB = { diff --git a/src/phase_ior_rnd4K.c b/src/phase_ior_rnd4K.c index 6e6b773..4584617 100644 --- a/src/phase_ior_rnd4K.c +++ b/src/phase_ior_rnd4K.c @@ -12,6 +12,7 @@ static ini_option_t option[] = { {"run", "Run this phase", 0, INI_BOOL, "TRUE", & ior_rnd4K_o.run}, {"verbosity", "The verbosity level", 0, INI_INT, 0, & ior_rnd4K_o.verbosity}, {"randomPrefill", "Prefill the file with this blocksize in bytes, e.g., 2097152", 0, INI_INT, "0", & ior_rnd4K_o.random_prefill_bytes}, + {"segmentCount", "Number of segments", 0, INI_INT, "10000000", & ior_rnd4K_o.segments}, {NULL} }; @@ -55,7 +56,7 @@ void ior_rnd4K_add_params(u_argv_t * argv){ u_argv_push(argv, "-k"); u_argv_push(argv, "-t=4096"); u_argv_push_printf(argv, "-b=%ld", d.block_size); - u_argv_push_printf(argv, "-s=%d", 10000000); + u_argv_push_printf(argv, "-s=%d", d.segments); /* number of segments */ } u_phase_t p_ior_rnd4K = { From 08301705b5d7a319b8b4742b63070dd8c8be1f6b Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 4 Jun 2022 11:20:05 +0200 Subject: [PATCH 07/21] Concurrent: Improved debugging, bugfix score calculation. --- src/phase_concurrent.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index 37f4a05..9002be5 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -12,7 +12,8 @@ typedef struct{ int run; char * command[CBENCHS]; mdworkbench_results_t * mdw; - IOR_point_t * ior; + IOR_point_t * iorr; + IOR_point_t * iorw; int color; // benchmark that is run } opt_concurrent_bench; @@ -101,6 +102,7 @@ static double run(void){ return 0; } double score = 0; + double time = 0; if(color == 0){ // run ior easy write if(ior_easy_o.filePerProc){ @@ -109,11 +111,13 @@ static double run(void){ u_purge_file(filename); } FILE * out = u_res_file_prep("concurrent-ior-easy-write", crank); - score = ior_process_write(argv[0], out, & o.ior, ccom); + score = ior_process_write(argv[0], out, & o.iorw, ccom); + time = o.iorw->time; }else if(color == 1){ // run ior rnd 1MB read FILE * out = u_res_file_prep("concurrent-rnd1MB-read", crank); - score = ior_process_read(argv[1], out, & o.ior, ccom); + score = ior_process_read(argv[1], out, & o.iorr, ccom); + time = o.iorr->time; }else if(color == 2){ // run md_workbench FILE * out = u_res_file_prep("concurrent-md-workbench", crank); @@ -122,27 +126,37 @@ static double run(void){ double rate = o.mdw->result[0].rate; PRINT_PAIR("maxOpTime", "%f\n", o.mdw->result[0].max_op_time); score = rate / 1000.0; + time = o.mdw->result[0].runtime; } if(crank == 0){ // exchange scores, calculate geometric mean score double scores[CBENCHS]; + double times[CBENCHS]; if(opt.rank == 0){ scores[0] = score; - double aggregated_score = score * workloads[0]; + times[0] = time; + double aggregated_score = score / workloads[0]; //printf("%f - %f - count: %d\n", score, aggregated_score, workloads[0]); for(int i=1; i < CBENCHS; i++){ UMPI_CHECK(MPI_Recv(& scores[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); - aggregated_score = aggregated_score * (scores[i] * (workloads[i] - workloads[i-1])); + UMPI_CHECK(MPI_Recv(& times[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); + aggregated_score = aggregated_score * (scores[i] / (workloads[i] - workloads[i-1])); //printf("%f - %f - count: %d\n", scores[i], aggregated_score, (workloads[i] - workloads[i-1])); } PRINT_PAIR("score-ior-easy-write", "%f\n", scores[0]); PRINT_PAIR("score-ior-rnd1MB-read", "%f\n", scores[1]); PRINT_PAIR("score-ior-md-workbench", "%f\n", scores[2]); - score = pow(aggregated_score / opt.mpi_size, 1.0/CBENCHS); + PRINT_PAIR("time-ior-easy-write", "%f\n", times[0]); + PRINT_PAIR("time-ior-rnd1MB-read", "%f\n", times[1]); + PRINT_PAIR("time-ior-md-workbench", "%f\n", times[2]); + score = pow(aggregated_score * opt.mpi_size, 1.0/CBENCHS); }else{ UMPI_CHECK(MPI_Send(& score, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD)); + UMPI_CHECK(MPI_Send(& time, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD)); } + }else{ + score = 0; } return score; } From 3506cdc317a72c016cf9701e2f50fe10eb79f162 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 4 Jun 2022 12:16:37 +0200 Subject: [PATCH 08/21] Enable flexible configuration of ration for testing. --- src/phase_concurrent.c | 77 ++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index 9002be5..7e28f20 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -15,23 +15,21 @@ typedef struct{ IOR_point_t * iorr; IOR_point_t * iorw; int color; // benchmark that is run + float ratio_write; + float ratio_read; } opt_concurrent_bench; static opt_concurrent_bench o; static ini_option_t option[] = { {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, - //{"", "XX", 0, INI_UINT, NULL, & o.XX}, + {"ratio-write", "The ratio of processes that perform write - this ratio has the highest priority", 0, INI_FLOAT, "0.2", & o.ratio_write}, + {"ratio-read", "The ratio of processes that perform random read - remaining processes will do metadata", 0, INI_FLOAT, "0.4", & o.ratio_read}, {NULL} }; static double run(void){ opt_concurrent_bench d = o; - - if(opt.mpi_size < 5){ - INVALID("The concurrent phase needs at least 5 processes\n"); - return 0; - } - + MPI_Comm ccom; /* Split processes into 3 groups and benchmarks: @@ -39,7 +37,22 @@ static double run(void){ benchmark 1 - 40% parallel read 1 MB - rnd1MB read benchmark 2 - 40% md-workbench for concurrent usage */ - int workloads[] = {opt.mpi_size * 0.2, opt.mpi_size * 0.6, opt.mpi_size}; + /* Sanity check of parameters */ + if(d.ratio_write > 1){ + d.ratio_write = 0; + } + if(d.ratio_read > 1){ + d.ratio_read = 0; + } + + int workloads[] = {opt.mpi_size * d.ratio_write, opt.mpi_size * (d.ratio_write+d.ratio_read), opt.mpi_size}; + int procs[] = {workloads[0], workloads[1] - workloads[0], workloads[2] - workloads[1]}; + + if(procs[0] + procs[1] == 0 || procs[0] + procs[2] == 0 || procs[1] + procs[2] == 0){ + INVALID("The concurrent phase doesn't make sense with two benchmarks using 0 processes\n"); + return 0; + } + int color = (opt.rank < workloads[0]) ? 0 : (opt.rank < workloads[1] ? 1 : 2); o.color = color; UMPI_CHECK(MPI_Comm_split(MPI_COMM_WORLD, color, opt.rank, & ccom)); @@ -47,7 +60,11 @@ static double run(void){ int crank; MPI_Comm_rank(ccom, & crank); /* printf("%d - %d - rank %d\n", opt.rank, color, crank); */ - PRINT_PAIR("benchmark", "%d\n", color); + if(opt.rank == 0){ + PRINT_PAIR("ranks-write", "%d\n", procs[0]); + PRINT_PAIR("ranks-read", "%d\n", procs[1]); + PRINT_PAIR("ranks-md", "%d\n", procs[2]); + } u_argv_t * argv[CBENCHS]; for(int i=0; i < CBENCHS; i++){ @@ -112,12 +129,16 @@ static double run(void){ } FILE * out = u_res_file_prep("concurrent-ior-easy-write", crank); score = ior_process_write(argv[0], out, & o.iorw, ccom); - time = o.iorw->time; + if(o.iorw){ + time = o.iorw->time; + } }else if(color == 1){ // run ior rnd 1MB read FILE * out = u_res_file_prep("concurrent-rnd1MB-read", crank); score = ior_process_read(argv[1], out, & o.iorr, ccom); - time = o.iorr->time; + if(o.iorr){ + time = o.iorr->time; + } }else if(color == 2){ // run md_workbench FILE * out = u_res_file_prep("concurrent-md-workbench", crank); @@ -126,23 +147,37 @@ static double run(void){ double rate = o.mdw->result[0].rate; PRINT_PAIR("maxOpTime", "%f\n", o.mdw->result[0].max_op_time); score = rate / 1000.0; - time = o.mdw->result[0].runtime; + if(o.mdw){ + time = o.mdw->result[0].runtime; + } } if(crank == 0){ // exchange scores, calculate geometric mean score - double scores[CBENCHS]; - double times[CBENCHS]; + double scores[CBENCHS] = {0}; + double times[CBENCHS] = {0}; if(opt.rank == 0){ - scores[0] = score; - times[0] = time; - double aggregated_score = score / workloads[0]; - //printf("%f - %f - count: %d\n", score, aggregated_score, workloads[0]); + MPI_Request req_score, req_time; + UMPI_CHECK(MPI_Isend(& score, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD, & req_score)); + UMPI_CHECK(MPI_Isend(& time, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD, & req_time)); + + double aggregated_score = 1; + int benchmarks = 0; + if (procs[0] != 0){ + scores[0] = score; + times[0] = time; + benchmarks++; + } for(int i=1; i < CBENCHS; i++){ + if (procs[i] == 0){ + scores[i] = 0; + times[i] = 0; + continue; + } + benchmarks++; UMPI_CHECK(MPI_Recv(& scores[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); UMPI_CHECK(MPI_Recv(& times[i], 1, MPI_DOUBLE, workloads[i-1], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE)); - aggregated_score = aggregated_score * (scores[i] / (workloads[i] - workloads[i-1])); - //printf("%f - %f - count: %d\n", scores[i], aggregated_score, (workloads[i] - workloads[i-1])); + aggregated_score = aggregated_score * (scores[i] / procs[i]); } PRINT_PAIR("score-ior-easy-write", "%f\n", scores[0]); PRINT_PAIR("score-ior-rnd1MB-read", "%f\n", scores[1]); @@ -150,7 +185,7 @@ static double run(void){ PRINT_PAIR("time-ior-easy-write", "%f\n", times[0]); PRINT_PAIR("time-ior-rnd1MB-read", "%f\n", times[1]); PRINT_PAIR("time-ior-md-workbench", "%f\n", times[2]); - score = pow(aggregated_score * opt.mpi_size, 1.0/CBENCHS); + score = pow(aggregated_score * opt.mpi_size, 1.0/benchmarks); }else{ UMPI_CHECK(MPI_Send(& score, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD)); UMPI_CHECK(MPI_Send(& time, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD)); From 207cfdd7fd7b731f35c8d44fb93a7da80b0e56ae Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 4 Jun 2022 13:23:45 +0200 Subject: [PATCH 09/21] Include min runtime for random reads. --- prepare.sh | 2 +- src/main.c | 1 + src/phase_concurrent.c | 7 +++---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/prepare.sh b/prepare.sh index e38cae6..c14ed72 100755 --- a/prepare.sh +++ b/prepare.sh @@ -7,7 +7,7 @@ echo It will also attempt to build the benchmarks echo It will output OK at the end if builds succeed echo -IOR_HASH=d3574d536643475269d37211e283b49ebd6732d7 +IOR_HASH=c4465beb21b6a82c7f3823d047df19096b6ab818 PFIND_HASH=62c3a7e31 INSTALL_DIR=$PWD diff --git a/src/main.c b/src/main.c index d30290e..f8a28cc 100644 --- a/src/main.c +++ b/src/main.c @@ -422,6 +422,7 @@ int main(int argc, char ** argv){ } if(! (phase->type & IO500_PHASE_DUMMY)){ PRINT_PAIR("score", "%f\n", score); + PRINT_PAIR("runtime", "%f\n", runtime); } char * valid_str = opt.is_valid_phase ? "" : " [INVALID]"; char score_str[40]; diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index 7e28f20..29d20e4 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -76,8 +76,7 @@ static double run(void){ opt_ior_easy d = ior_easy_o; ior_easy_add_params(argv[0], 0); u_argv_push(argv[0], "-w"); /* write file */ - u_argv_push(argv[0], "-D"); /* deadline for stonewall in seconds */ - u_argv_push_printf(argv[0], "%d", opt.stonewall); + u_argv_push_printf(argv[0], "-D=%d", opt.stonewall); /* deadline for stonewall in seconds */ //u_argv_push_default_if_set_api_options(argv[0], "-a", d.api, o.api); u_argv_push_default_if_set_api_options(argv[0], "-a", d.api, d.api); // TODO USE RND WRITE OPTION u_argv_push(argv[0], "-O"); @@ -94,8 +93,8 @@ static double run(void){ u_argv_push_default_if_set_api_options(argv[1], "-a", d.api, d.api); // TODO USE RND WRITE OPTION u_argv_push(argv[1], "-O"); u_argv_push_printf(argv[1], "saveRankPerformanceDetailsCSV=%s/concurrent-ior-rnd1MB-read.csv", opt.resdir); - u_argv_push(argv[0], "-D"); /* deadline for stonewall in seconds */ - u_argv_push_printf(argv[0], "%d", opt.stonewall); + u_argv_push_printf(argv[1], "-D=%d", opt.stonewall); /* deadline for stonewall in seconds */ + u_argv_push_printf(argv[1], "-O=minTimeDuration=%d", opt.stonewall); /* minimum runtime */ o.command[1] = u_flatten_argv(argv[1]); PRINT_PAIR("exe-rnd1MB-read", "%s\n", o.command[1]); } From 5b2898c08e728a9dc5be03789dcb7dea8f661312 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 4 Jun 2022 13:27:43 +0200 Subject: [PATCH 10/21] MinTime for write too... --- src/phase_concurrent.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index 29d20e4..5b3a122 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -77,6 +77,7 @@ static double run(void){ ior_easy_add_params(argv[0], 0); u_argv_push(argv[0], "-w"); /* write file */ u_argv_push_printf(argv[0], "-D=%d", opt.stonewall); /* deadline for stonewall in seconds */ + u_argv_push_printf(argv[0], "-O=minTimeDuration=%d", opt.stonewall); /* minimum runtime */ //u_argv_push_default_if_set_api_options(argv[0], "-a", d.api, o.api); u_argv_push_default_if_set_api_options(argv[0], "-a", d.api, d.api); // TODO USE RND WRITE OPTION u_argv_push(argv[0], "-O"); From 96f34fa97be2b5d6e8ed23d3439271443c9dfa8a Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 4 Jun 2022 16:21:34 +0200 Subject: [PATCH 11/21] Increased default verbosity. --- src/main.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/main.c b/src/main.c index f8a28cc..1449dbe 100644 --- a/src/main.c +++ b/src/main.c @@ -26,6 +26,13 @@ static char const * io500_phase_str[IO500_SCORE_LAST] = { "CONCURRENT" }; +static char const * io500_unit_str[IO500_SCORE_LAST] = { + "", + "kIOPS", + "GiB/s", + "" +}; + static void prepare_aiori(void){ // check selected API, might be followed by API options char * api = strdup(opt.api); @@ -390,11 +397,9 @@ int main(int argc, char ** argv){ MPI_Barrier(MPI_COMM_WORLD); if(opt.rank == 0){ fprintf(file_out, "\n[%s]\n", phase->name); - if(opt.verbosity > 0){ - PRINT_PAIR_HEADER("t_start"); - u_print_timestamp(file_out); - fprintf(file_out, "\n"); - } + PRINT_PAIR_HEADER("t_start"); + u_print_timestamp(file_out); + fprintf(file_out, "\n"); } double start = GetTimeStamp(); @@ -422,7 +427,6 @@ int main(int argc, char ** argv){ } if(! (phase->type & IO500_PHASE_DUMMY)){ PRINT_PAIR("score", "%f\n", score); - PRINT_PAIR("runtime", "%f\n", runtime); } char * valid_str = opt.is_valid_phase ? "" : " [INVALID]"; char score_str[40]; @@ -432,7 +436,12 @@ int main(int argc, char ** argv){ }else{ dupprintf("[ ]"); } - dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, phase->name[0] == 'i' ? "GiB/s" : "kIOPS", runtime, valid_str); + dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, io500_unit_str[phase->type], runtime, valid_str); + + PRINT_PAIR("t_delta", "%.4f\n", runtime); + PRINT_PAIR_HEADER("t_end"); + u_print_timestamp(file_out); + fprintf(file_out, "\n"); } if(phase->group > IO500_NO_SCORE){ if(phase->type & IO500_PHASE_FLAG_OPTIONAL){ @@ -442,14 +451,6 @@ int main(int argc, char ** argv){ } } phases[i]->score = score; - - - if(opt.verbosity > 0 && opt.rank == 0){ - PRINT_PAIR("t_delta", "%.4f\n", runtime); - PRINT_PAIR_HEADER("t_end"); - u_print_timestamp(file_out); - fprintf(file_out, "\n"); - } } MPI_Barrier(MPI_COMM_WORLD); From 0b8df64b7fb5bcc653a97cfb4f59d7e9fe985547 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 4 Jun 2022 20:29:01 +0200 Subject: [PATCH 12/21] Utilize pfind for deletion of mdworkbench to cleanup properly - supply -e -E flag to find. --- find/sfind.sh | 19 +++++++++- prepare.sh | 4 +- src/main.c | 6 +-- src/phase_concurrent.c | 2 +- src/phase_find.c | 6 +-- src/phase_find_easy.c | 2 +- src/phase_find_hard.c | 4 +- src/phase_mdworkbench_delete.c | 69 +++++++++++++++++++++++++++------- 8 files changed, 85 insertions(+), 27 deletions(-) diff --git a/find/sfind.sh b/find/sfind.sh index b76e8fd..e7bf349 100755 --- a/find/sfind.sh +++ b/find/sfind.sh @@ -9,6 +9,10 @@ # When you are done pring 'x/y' where x is matched files and y is total files searched. # There is a parallel version also install in bin that might be better +# The script should support the following extra options +# -e for deletion of found files +# -E for deletion of directories + function parse_rates { #find -D rates gives a weird thing like this: #Predicate success rates after completion @@ -18,7 +22,20 @@ function parse_rates { echo $rates | tr " " "\n" | grep '/' | $1 -1 | cut -d \/ -f 2 | cut -d = -f 1 } -rates=`find -D rates $* 2>&1 | grep -A1 Predicate | tail -1` +IN="" +for var in "$@" +do + if [[ $var == "-e" ]] ; then + continue + fi + if [[ $var == "-E" ]] ; then + IN="$IN -exec rm -f {} ;" + continue + fi + IN="$IN $var" +done + +rates=$(find -D rates $IN 2>&1 | grep -A1 Predicate | tail -1) total_files=$(parse_rates 'head') match_files=$(parse_rates 'tail') echo "MATCHED $match_files/$total_files" diff --git a/prepare.sh b/prepare.sh index c14ed72..9effc60 100755 --- a/prepare.sh +++ b/prepare.sh @@ -7,8 +7,8 @@ echo It will also attempt to build the benchmarks echo It will output OK at the end if builds succeed echo -IOR_HASH=c4465beb21b6a82c7f3823d047df19096b6ab818 -PFIND_HASH=62c3a7e31 +IOR_HASH=c4465beb21b6a +PFIND_HASH=d97bcb944f4 INSTALL_DIR=$PWD BIN=$INSTALL_DIR/bin diff --git a/src/main.c b/src/main.c index 1449dbe..9752b8f 100644 --- a/src/main.c +++ b/src/main.c @@ -27,10 +27,10 @@ static char const * io500_phase_str[IO500_SCORE_LAST] = { }; static char const * io500_unit_str[IO500_SCORE_LAST] = { - "", + " ", "kIOPS", "GiB/s", - "" + "score" }; static void prepare_aiori(void){ @@ -436,7 +436,7 @@ int main(int argc, char ** argv){ }else{ dupprintf("[ ]"); } - dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, io500_unit_str[phase->type], runtime, valid_str); + dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, io500_unit_str[phase->group], runtime, valid_str); PRINT_PAIR("t_delta", "%.4f\n", runtime); PRINT_PAIR_HEADER("t_end"); diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index 5b3a122..9e73e8f 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -185,7 +185,7 @@ static double run(void){ PRINT_PAIR("time-ior-easy-write", "%f\n", times[0]); PRINT_PAIR("time-ior-rnd1MB-read", "%f\n", times[1]); PRINT_PAIR("time-ior-md-workbench", "%f\n", times[2]); - score = pow(aggregated_score * opt.mpi_size, 1.0/benchmarks); + score = pow(aggregated_score, 1.0/benchmarks) * opt.mpi_size; }else{ UMPI_CHECK(MPI_Send(& score, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD)); UMPI_CHECK(MPI_Send(& time, 1, MPI_DOUBLE, 0, 0, MPI_COMM_WORLD)); diff --git a/src/phase_find.c b/src/phase_find.c index 1a2f009..85390ea 100644 --- a/src/phase_find.c +++ b/src/phase_find.c @@ -98,7 +98,7 @@ double run_find(const char * phase_name, opt_find * of){ ERROR("Failed to run find command: \"%s\" error: %s\n", of->command, strerror(errno)); return -1; } - char line[1024]; + char line[PATH_MAX]; uint64_t hits = 0; *line = '\0'; while (fgets(line, sizeof(line), fp) != NULL) { @@ -161,7 +161,7 @@ static ini_option_t option[] = { static void validate(void){ if(of.run == 0) return; if(of.ext_find){ - char args[2048]; + char args[PATH_MAX]; sprintf(args, "%s -newer %s/timestampfile -size 3901c -name \"*01*\"", opt.datadir, opt.resdir); external_find_prepare_arguments(args, & of); }else{ @@ -198,7 +198,7 @@ void external_find_prepare_arguments(char * args, opt_find * of){ if(! (sb.st_mode & S_IXUSR) ){ FATAL("The external-script must be a executable file %s\n", of->ext_find); } - char command[2048]; + char command[PATH_MAX]; sprintf(command, "%s %s %s %s", of->ext_mpi, of->ext_find, of->ext_args, args); of->command = strdup(command); diff --git a/src/phase_find_easy.c b/src/phase_find_easy.c index b2f3236..1fcf973 100644 --- a/src/phase_find_easy.c +++ b/src/phase_find_easy.c @@ -29,7 +29,7 @@ static ini_option_t option[] = { static void validate(void){ if(of.run == 0) return; if(of.ext_find){ - char args[1024]; + char args[PATH_MAX]; sprintf(args, "%s/mdtest-easy/ -name \"*01*\"", opt.datadir); external_find_prepare_arguments(args, & of); }else{ diff --git a/src/phase_find_hard.c b/src/phase_find_hard.c index 2481a32..9b7e517 100644 --- a/src/phase_find_hard.c +++ b/src/phase_find_hard.c @@ -12,7 +12,7 @@ static opt_find of; static double run(void){ if(of.run == 0) return 0.0; - return run_find("find-hard", & of);; + return run_find("find-hard", & of); } static ini_option_t option[] = { @@ -29,7 +29,7 @@ static ini_option_t option[] = { static void validate(void){ if(of.run == 0) return; if(of.ext_find){ - char args[1024]; + char args[PATH_MAX]; sprintf(args, "%s/mdtest-hard/ -newer %s/timestampfile -size 3901c -name \"*01*\"", opt.datadir, opt.resdir); external_find_prepare_arguments(args, & of); }else{ diff --git a/src/phase_mdworkbench_delete.c b/src/phase_mdworkbench_delete.c index 2ba073e..96bcf00 100644 --- a/src/phase_mdworkbench_delete.c +++ b/src/phase_mdworkbench_delete.c @@ -4,6 +4,8 @@ #include #include +#include +#include typedef struct{ int run; @@ -12,30 +14,69 @@ typedef struct{ } opt_mdworkbench_delete; static opt_mdworkbench_delete o; +static opt_find of; static ini_option_t option[] = { {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, + {"external-script", "Set to an external script to perform the find phase", 0, INI_STRING, NULL, & of.ext_find}, + {"external-mpi-args", "Startup arguments for external scripts, some MPI's may not support this!", 0, INI_STRING, "", & of.ext_mpi}, + {"external-extra-args", "Extra arguments for the external scripts", 0, INI_STRING, "", & of.ext_args}, + {"nproc", "Set the number of processes for pfind/the external script", 0, INI_UINT, NULL, & of.nproc}, {NULL} }; +static void validate(void){ +} + static double run(void){ opt_mdworkbench d = mdworkbench_o; - u_argv_t * argv = u_argv_create(); - mdworkbench_add_params(argv, 0); - u_argv_push(argv, "-3"); - - o.command = u_flatten_argv(argv); - PRINT_PAIR("exe", "%s\n", o.command); + if(1){ + u_argv_push(argv, "./pfind"); + u_argv_push_printf(argv, "%s/mdworkbench", opt.datadir); + u_argv_push(argv, "-C"); + u_argv_push(argv, "-E"); + u_argv_push(argv, "-e"); + o.command = u_flatten_argv(argv); + of.pfind_o = pfind_parse_args(argv->size, argv->vector, 0, MPI_COMM_WORLD); + }else{ + mdworkbench_add_params(argv, 0); + u_argv_push(argv, "-3"); + o.command = u_flatten_argv(argv); + } + if(opt.dry_run || d.run == 0){ - u_argv_free(argv); return 0; + } + if(1){ + if(of.ext_find){ + char args[PATH_MAX]; + sprintf(args, "%s/mdworkbench/ -E -e", opt.datadir); + external_find_prepare_arguments(args, & of); + return run_find(p_mdworkbench_delete.name, & of); + }else{ + pfind_find_results_t * res = pfind_find(of.pfind_o); + if(! res){ + WARNING("Pfind cleanup returned with an error.\n") + return 0.0; + } + of.pfind_res = pfind_aggregrate_results(res); + of.found_files = of.pfind_res->found_files; + of.runtime = of.pfind_res->runtime; + if(opt.rank == 0){ + PRINT_PAIR("deleted", "%"PRIu64"\n", of.found_files); + return of.pfind_res->total_files / of.runtime / 1000; + } + return 0.0; + } + }else{ + FILE * out = u_res_file_prep(p_mdworkbench_delete.name, opt.rank); + mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD); + PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); + + double rate = o.res->result[0].rate; + return rate / 1000.0; } - FILE * out = u_res_file_prep(p_mdworkbench_delete.name, opt.rank); - mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD); - PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); - - double rate = o.res->result[0].rate; - return rate / 1000.0; + return 0.0; } @@ -43,7 +84,7 @@ u_phase_t p_mdworkbench_delete = { "mdworkbench-delete", IO500_PHASE_REMOVE | IO500_PHASE_FLAG_OPTIONAL, option, - NULL, + validate, run, 0, .group = IO500_NO_SCORE, From 40e550f7d8341deead2ba2571120a64ad39bdebf Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 5 Jun 2022 11:12:40 +0200 Subject: [PATCH 13/21] Refactoring of find common optimization code. Moved mdw original code into separate file. --- src/main.c | 2 +- src/phase_find.c | 21 +++++---- src/phase_find_easy.c | 9 ---- src/phase_find_hard.c | 9 ---- src/phase_mdworkbench_delete.c | 74 ++++++++---------------------- src/phase_mdworkbench_delete_mdw.c | 50 ++++++++++++++++++++ 6 files changed, 80 insertions(+), 85 deletions(-) create mode 100644 src/phase_mdworkbench_delete_mdw.c diff --git a/src/main.c b/src/main.c index 9752b8f..ec70d9b 100644 --- a/src/main.c +++ b/src/main.c @@ -436,7 +436,7 @@ int main(int argc, char ** argv){ }else{ dupprintf("[ ]"); } - dupprintf(" %20s %15s %s : time %.3f seconds%s\n", phase->name, score_str, io500_unit_str[phase->group], runtime, valid_str); + dupprintf(" %25s %15s %s : time %.3f seconds%s\n", phase->name, score_str, io500_unit_str[phase->group], runtime, valid_str); PRINT_PAIR("t_delta", "%.4f\n", runtime); PRINT_PAIR_HEADER("t_end"); diff --git a/src/phase_find.c b/src/phase_find.c index 85390ea..8537294 100644 --- a/src/phase_find.c +++ b/src/phase_find.c @@ -175,16 +175,7 @@ static void validate(void){ u_argv_push(argv, "-name"); u_argv_push(argv, "*01*"); u_argv_push(argv, "-C"); - if(of.pfind_steal_from_next){ - u_argv_push(argv, "-N"); - } - if(of.pfind_par_single_dir_access_hash){ - u_argv_push(argv, "-H"); - u_argv_push(argv, "1"); - } - u_argv_push(argv, "-q"); - u_argv_push_printf(argv, "%d", of.pfind_queue_length); - + pfind_prepare_arguments(argv, & of); } } @@ -226,6 +217,16 @@ void pfind_prepare_arguments(u_argv_t * argv, opt_find * of){ com = MPI_COMM_NULL; } } + + if(of->pfind_steal_from_next){ + u_argv_push(argv, "-N"); + } + if(of->pfind_par_single_dir_access_hash){ + u_argv_push(argv, "-H"); + u_argv_push(argv, "1"); + } + u_argv_push(argv, "-q"); + u_argv_push_printf(argv, "%d", of->pfind_queue_length); of->command = u_flatten_argv(argv); of->pfind_com = com; diff --git a/src/phase_find_easy.c b/src/phase_find_easy.c index 1fcf973..52f49c2 100644 --- a/src/phase_find_easy.c +++ b/src/phase_find_easy.c @@ -39,15 +39,6 @@ static void validate(void){ u_argv_push(argv, "-name"); u_argv_push(argv, "*01*"); u_argv_push(argv, "-C"); - if(of.pfind_steal_from_next){ - u_argv_push(argv, "-N"); - } - if(of.pfind_par_single_dir_access_hash){ - u_argv_push(argv, "-H"); - u_argv_push(argv, "1"); - } - u_argv_push(argv, "-q"); - u_argv_push_printf(argv, "%d", of.pfind_queue_length); pfind_prepare_arguments(argv, & of); } diff --git a/src/phase_find_hard.c b/src/phase_find_hard.c index 9b7e517..5eaf119 100644 --- a/src/phase_find_hard.c +++ b/src/phase_find_hard.c @@ -43,15 +43,6 @@ static void validate(void){ u_argv_push(argv, "-name"); u_argv_push(argv, "*01*"); u_argv_push(argv, "-C"); - if(of.pfind_steal_from_next){ - u_argv_push(argv, "-N"); - } - if(of.pfind_par_single_dir_access_hash){ - u_argv_push(argv, "-H"); - u_argv_push(argv, "1"); - } - u_argv_push(argv, "-q"); - u_argv_push_printf(argv, "%d", of.pfind_queue_length); pfind_prepare_arguments(argv, & of); } diff --git a/src/phase_mdworkbench_delete.c b/src/phase_mdworkbench_delete.c index 96bcf00..e543a9d 100644 --- a/src/phase_mdworkbench_delete.c +++ b/src/phase_mdworkbench_delete.c @@ -3,89 +3,51 @@ #include #include -#include #include #include -typedef struct{ - int run; - char * command; - mdworkbench_results_t * res; -} opt_mdworkbench_delete; - -static opt_mdworkbench_delete o; static opt_find of; static ini_option_t option[] = { - {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, {"external-script", "Set to an external script to perform the find phase", 0, INI_STRING, NULL, & of.ext_find}, {"external-mpi-args", "Startup arguments for external scripts, some MPI's may not support this!", 0, INI_STRING, "", & of.ext_mpi}, {"external-extra-args", "Extra arguments for the external scripts", 0, INI_STRING, "", & of.ext_args}, {"nproc", "Set the number of processes for pfind/the external script", 0, INI_UINT, NULL, & of.nproc}, + {"run", "Run this phase", 0, INI_BOOL, "TRUE", & of.run}, + {"pfind-queue-length", "Pfind queue length", 0, INI_INT, "10000", & of.pfind_queue_length}, + {"pfind-steal-next", "Pfind Steal from next", 0, INI_BOOL, "FALSE", & of.pfind_steal_from_next}, + {"pfind-parallelize-single-dir-access-using-hashing", "Parallelize the readdir by using hashing. Your system must support this!", 0, INI_BOOL, "FALSE", & of.pfind_par_single_dir_access_hash}, {NULL} }; static void validate(void){ -} - -static double run(void){ - opt_mdworkbench d = mdworkbench_o; - u_argv_t * argv = u_argv_create(); - if(1){ + if(of.ext_find){ + char args[PATH_MAX]; + sprintf(args, "%s/mdworkbench/ -E -e", opt.datadir); + external_find_prepare_arguments(args, & of); + }else{ + u_argv_t * argv = u_argv_create(); u_argv_push(argv, "./pfind"); u_argv_push_printf(argv, "%s/mdworkbench", opt.datadir); u_argv_push(argv, "-C"); u_argv_push(argv, "-E"); u_argv_push(argv, "-e"); - o.command = u_flatten_argv(argv); - of.pfind_o = pfind_parse_args(argv->size, argv->vector, 0, MPI_COMM_WORLD); - }else{ - mdworkbench_add_params(argv, 0); - u_argv_push(argv, "-3"); - o.command = u_flatten_argv(argv); - } - - if(opt.dry_run || d.run == 0){ - return 0; - } - if(1){ - if(of.ext_find){ - char args[PATH_MAX]; - sprintf(args, "%s/mdworkbench/ -E -e", opt.datadir); - external_find_prepare_arguments(args, & of); - return run_find(p_mdworkbench_delete.name, & of); - }else{ - pfind_find_results_t * res = pfind_find(of.pfind_o); - if(! res){ - WARNING("Pfind cleanup returned with an error.\n") - return 0.0; - } - of.pfind_res = pfind_aggregrate_results(res); - of.found_files = of.pfind_res->found_files; - of.runtime = of.pfind_res->runtime; - if(opt.rank == 0){ - PRINT_PAIR("deleted", "%"PRIu64"\n", of.found_files); - return of.pfind_res->total_files / of.runtime / 1000; - } - return 0.0; - } - }else{ - FILE * out = u_res_file_prep(p_mdworkbench_delete.name, opt.rank); - mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD); - PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); - double rate = o.res->result[0].rate; - return rate / 1000.0; + pfind_prepare_arguments(argv, & of); } - return 0.0; +} + +static double run(void){ + if(opt.dry_run || of.run == 0) return 0.0; + return run_find(p_mdworkbench_delete.name, & of); } u_phase_t p_mdworkbench_delete = { - "mdworkbench-delete", + "mdworkbench-find-delete", IO500_PHASE_REMOVE | IO500_PHASE_FLAG_OPTIONAL, option, validate, run, 0, - .group = IO500_NO_SCORE, + .group = IO500_SCORE_MD, }; diff --git a/src/phase_mdworkbench_delete_mdw.c b/src/phase_mdworkbench_delete_mdw.c new file mode 100644 index 0000000..b7df011 --- /dev/null +++ b/src/phase_mdworkbench_delete_mdw.c @@ -0,0 +1,50 @@ +#include +#include +#include + +#include +#include + +typedef struct{ + int run; + char * command; + mdworkbench_results_t * res; +} opt_mdworkbench_delete; + +static opt_mdworkbench_delete o; + +static ini_option_t option[] = { + {"run", "Run this phase", 0, INI_BOOL, "TRUE", & o.run}, + {NULL} }; + +static void validate(void){ +} + +static double run(void){ + opt_mdworkbench d = mdworkbench_o; + u_argv_t * argv = u_argv_create(); + mdworkbench_add_params(argv, 0); + u_argv_push(argv, "-3"); + o.command = u_flatten_argv(argv); + + if(opt.dry_run || d.run == 0){ + return 0; + } + FILE * out = u_res_file_prep(p_mdworkbench_delete.name, opt.rank); + mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD); + PRINT_PAIR("maxOpTime", "%f\n", o.res->result[0].max_op_time); + + double rate = o.res->result[0].rate; + return rate / 1000.0; +} + + +u_phase_t p_mdworkbench_delete = { + "mdworkbench-delete", + IO500_PHASE_REMOVE | IO500_PHASE_FLAG_OPTIONAL, + option, + validate, + run, + 0, + .group = IO500_NO_SCORE, +}; From 61d5ed6626e4c34222e28a4d26c51db8e0592ff7 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 5 Jun 2022 12:06:16 +0200 Subject: [PATCH 14/21] Output pretty file with data ordered into logical structures. --- src/main.c | 114 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 109 insertions(+), 5 deletions(-) diff --git a/src/main.c b/src/main.c index ec70d9b..58e9ce3 100644 --- a/src/main.c +++ b/src/main.c @@ -138,6 +138,7 @@ static double calc_score(double scores[IO500_SCORE_LAST], int extended, uint32_t if(extended){ groups = IO500_SCORE_CONCURRENT; } + memset(scores, 0, sizeof(double) * IO500_SCORE_LAST); for(io500_phase_score_group g=1; g <= groups; g++){ char score_string[2048]; char *p = score_string; @@ -180,6 +181,92 @@ static const char * io500_mode_str(io500_mode mode){ } } +typedef struct { + int run; + int valid; + double score; + double time; +} io500_phase_result_t; + +typedef struct{ + double score[IO500_SCORE_LAST]; + double scoreX[IO500_SCORE_LAST]; + double runtime; +} io500_overall_results_t; + +static io500_phase_result_t io500_results[IO500_PHASES]; +static io500_overall_results_t io500_overall_results; + +static void dump_io500_phase(FILE * o, int i){ + io500_phase_result_t * r = & io500_results[i]; + if(! r->run) return; + u_phase_t * phase = phases[i]; + char score_str[40]; + sprintf(score_str, "%f", r->score); + fprintf(o, "%25s\t%15s\t%s\t%.3fs\t%s\n", phase->name, score_str, io500_unit_str[phase->group], r->time, r->valid ? "" : "INVALID"); +} + +static void dump_io500_results(void){ + FILE * res; + char file[PATH_MAX]; + sprintf(file, "%s/result_pretty.txt", opt.resdir); + res = fopen(file, "w"); + if(! res){ + FATAL("Could not open \"%s\" for writing (%s)\n", file, strerror(errno)); + } + + fprintf(res, "%25s\t%fs\n", "RUNTIME", io500_overall_results.runtime); + fprintf(res, "%25s", "SCORE"); + for(int i=0; i < IO500_SCORE_LAST; i++){ + fprintf(res, "\t%f\t%s\t", io500_overall_results.score[i], io500_unit_str[i]); + } + fprintf(res, "\n"); + + if(opt.mode == IO500_MODE_EXTENDED){ + fprintf(res, "%25s", "SCOREX"); + for(int i=0; i < IO500_SCORE_LAST; i++){ + fprintf(res, "\t%f\t%s\t", io500_overall_results.scoreX[i], io500_unit_str[i]); + } + fprintf(res, "\n"); + } + + // ior easy + dump_io500_phase(res, 3); + dump_io500_phase(res, 24); + // ior hard + dump_io500_phase(res, 15); + dump_io500_phase(res, 26); + // rand4k + dump_io500_phase(res, 5); + dump_io500_phase(res, 19); + // rand1M + dump_io500_phase(res, 8); + dump_io500_phase(res, 20); + // concurrent + dump_io500_phase(res, 23); + + // MD easy + dump_io500_phase(res, 7); + dump_io500_phase(res, 25); + dump_io500_phase(res, 29); + // MD hard + dump_io500_phase(res, 17); + dump_io500_phase(res, 27); + dump_io500_phase(res, 30); + dump_io500_phase(res, 31); + // MDW + dump_io500_phase(res, 22); + dump_io500_phase(res, 28); + // find + dump_io500_phase(res, 18); + // find easy + dump_io500_phase(res, 13); + // find hard + dump_io500_phase(res, 21); + + fclose(res); +} + int main(int argc, char ** argv){ int mpi_init = 0; file_out = stdout; @@ -212,6 +299,9 @@ int main(int argc, char ** argv){ } goto out; } + + memset(io500_results, 0, sizeof(io500_results)); + double t_io500_start = GetTimeStamp(); mpi_init = 1; MPI_Init(& argc, & argv); @@ -314,7 +404,7 @@ int main(int argc, char ** argv){ FILE * res_summary = NULL; if(opt.rank == 0){ - char file[2048]; + char file[PATH_MAX]; sprintf(file, "%s/result_summary.txt", opt.resdir); res_summary = fopen(file, "w"); if(! res_summary){ @@ -340,7 +430,7 @@ int main(int argc, char ** argv){ if(opt.rank == 0){ // create configuration in result directory to ensure it is preserved - char file[2048]; + char file[PATH_MAX]; sprintf(file, "%s/config.ini", opt.resdir); FILE * fd = fopen(file, "w"); if(! fd){ @@ -379,11 +469,12 @@ int main(int argc, char ** argv){ for(int i=0; i < IO500_PHASES; i++){ u_phase_t * phase = phases[i]; - if(! phase->run) continue; + if(! phase->run) continue; if( cleanup_only && ! (phase->type & IO500_PHASE_REMOVE) ) continue; if(! RUN_PHASE(phase)){ continue; } + io500_results[i].run = 1; if(opt.drop_caches && ! (phase->type & IO500_PHASE_DUMMY) ){ DEBUG_INFO("Dropping cache\n"); @@ -425,6 +516,11 @@ int main(int argc, char ** argv){ opt.is_valid_run = 0; } } + + io500_results[i].score = score; + io500_results[i].time = runtime; + io500_results[i].valid = opt.is_valid_phase; + if(! (phase->type & IO500_PHASE_DUMMY)){ PRINT_PAIR("score", "%f\n", score); } @@ -468,7 +564,10 @@ int main(int argc, char ** argv){ PRINT_PAIR("hash", "%X\n", (int) score_hash); dupprintf("[SCORE ] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n", scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_score, valid_str); - + + scores[IO500_NO_SCORE] = overall_score; + memcpy(io500_overall_results.score, scores, sizeof(scores)); + // extended run if(opt.mode == IO500_MODE_EXTENDED){ valid_str = opt.is_valid_extended_run ? "" : " [INVALID]"; @@ -481,8 +580,13 @@ int main(int argc, char ** argv){ PRINT_PAIR("SCORE", "%f%s\n", overall_extended_score, valid_str); PRINT_PAIR("hash", "%X\n", (int) score_extended_hash); dupprintf("[SCOREX] Bandwidth %f GiB/s : IOPS %f kiops : TOTAL %f%s\n", scores[IO500_SCORE_BW], scores[IO500_SCORE_MD], overall_extended_score, valid_str); + + scores[IO500_NO_SCORE] = overall_score; + memcpy(io500_overall_results.scoreX, scores, sizeof(scores)); } - + + io500_overall_results.runtime = GetTimeStamp() - t_io500_start; + dump_io500_results(); printf("\nThe result files are stored in the directory: %s\n", opt.resdir); } From a62cce0b1169ab4193f1aa61b2d7f1948e4d7978 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 5 Jun 2022 13:33:38 +0200 Subject: [PATCH 15/21] Utilize MDTest error message. --- prepare.sh | 2 +- src/phase_mdtest.c | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/prepare.sh b/prepare.sh index 9effc60..eb42add 100755 --- a/prepare.sh +++ b/prepare.sh @@ -7,7 +7,7 @@ echo It will also attempt to build the benchmarks echo It will output OK at the end if builds succeed echo -IOR_HASH=c4465beb21b6a +IOR_HASH=07e2feb6e0e9a2 PFIND_HASH=d97bcb944f4 INSTALL_DIR=$PWD diff --git a/src/phase_mdtest.c b/src/phase_mdtest.c index 2b8b2e3..8428f1f 100644 --- a/src/phase_mdtest.c +++ b/src/phase_mdtest.c @@ -7,6 +7,10 @@ void p_mdtest_run(u_argv_t * argv, FILE * out, mdtest_generic_res * d, mdtest_te d->items = res->items[test]; d->rate = res->rate[test] / 1000; d->rate_stonewall = res->stonewall_item_sum[test] / res->stonewall_time[test] / 1000; + INFO_PAIR("errors", "%.0f\n", (float) res->total_errors); + if(res->total_errors){ + INVALID("Verification errors %.0f \n", (float) res->total_errors); + } u_res_file_close(out); u_argv_free(argv); free(res); From c8b08346b87fbe2bcb2176abde30f2d3bc1b85e8 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 5 Jun 2022 13:41:56 +0200 Subject: [PATCH 16/21] Analyze MDworkbench results. --- src/phase_mdworkbench_bench.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/phase_mdworkbench_bench.c b/src/phase_mdworkbench_bench.c index bd68cd9..c9881c7 100644 --- a/src/phase_mdworkbench_bench.c +++ b/src/phase_mdworkbench_bench.c @@ -36,9 +36,12 @@ static double run(void){ FILE * out = u_res_file_prep(p_mdworkbench_bench.name, opt.rank); mdworkbench_process(argv, out, & o.res, MPI_COMM_WORLD); if(o.res->count != 2){ - INVALID("During the md-workbench phase not two iterations are performed but %d This invalidates your run.\n", o.res->count); + INVALID("During the md-workbench phase not two iterations are performed but %d. This invalidates your run.\n", o.res->count); return 0.0; } + if(o.res->errors){ + INVALID("md-workbench reported %lld errors\n", (long long int) o.res->errors); + } double rate = o.res->result[1].rate; if( o.res->result[0].rate < rate * 0.5 From 2a4b7e716e3745e2bbdfca4fcd3c8d0577b84a72 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 11 Jun 2022 20:42:36 +0200 Subject: [PATCH 17/21] PFind removed bogus CSV output. --- src/phase_find.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/phase_find.c b/src/phase_find.c index 8537294..c7e31af 100644 --- a/src/phase_find.c +++ b/src/phase_find.c @@ -60,7 +60,7 @@ double run_find(const char * phase_name, opt_find * of){ fprintf(fd, "runtime: %f rate: %f\n", of->pfind_res->runtime, of->pfind_res->rate); fprintf(fd, "rank, errors, unknown, found, total, checked, job steal msgs received, work items send, job steal msgs send, work items stolen, time spend in job stealing in s, number of completion tokens send\n"); fprintf(fd, "0, %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64, res->errors, res->unknown_file, res->found_files, res->total_files, res->checked_dirents); - fprintf(fd, ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %.3fs, %"PRIu64"\n", res->monitor.job_steal_inbound, res->monitor.work_send, res->monitor.job_steal_tries, res->monitor.work_stolen, res->monitor.job_steal_mpitime_us / 1000000.0, res->monitor.completion_tokens_send); + fprintf(fd, ", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %.3f, %"PRIu64"\n", res->monitor.job_steal_inbound, res->monitor.work_send, res->monitor.job_steal_tries, res->monitor.work_stolen, res->monitor.job_steal_mpitime_us / 1000000.0, res->monitor.completion_tokens_send); for(int i=1; i < of->nproc; i++){ MPI_Recv(& res->errors, 5, MPI_LONG_LONG_INT, i, 4712, of->pfind_com, MPI_STATUS_IGNORE); fprintf(fd, "%d, %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64, i, res->errors, res->unknown_file, res->found_files, res->total_files, res->checked_dirents); From fc1ed848e1d5d69a78f3e83f6d982cb535bb7704 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Mon, 13 Jun 2022 18:29:38 +0200 Subject: [PATCH 18/21] Disable read check for concurrent. --- src/phase_concurrent.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index 9e73e8f..b22fea8 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -90,7 +90,6 @@ static double run(void){ opt_ior_rnd d = ior_rnd1MB_o; ior_rnd1MB_add_params(argv[1]); u_argv_push(argv[1], "-r"); - u_argv_push(argv[1], "-R"); u_argv_push_default_if_set_api_options(argv[1], "-a", d.api, d.api); // TODO USE RND WRITE OPTION u_argv_push(argv[1], "-O"); u_argv_push_printf(argv[1], "saveRankPerformanceDetailsCSV=%s/concurrent-ior-rnd1MB-read.csv", opt.resdir); From 1af6670508d40a9d1881560a1e1caf42f63181cd Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 24 Dec 2022 14:45:57 +0100 Subject: [PATCH 19/21] Port to enhanced IOR GPU support. --- include/io500-opt.h | 3 ++- src/phase_ior_easy.c | 7 +++++-- src/phase_ior_hard.c | 7 +++++-- src/phase_ior_rnd1MB.c | 7 +++++-- src/phase_ior_rnd4K.c | 7 +++++-- src/phase_mdtest_easy.c | 7 +++++-- src/phase_mdtest_hard.c | 8 ++++++-- src/phase_mdworkbench.c | 9 ++++++--- src/phase_opt.c | 3 ++- 9 files changed, 41 insertions(+), 17 deletions(-) diff --git a/include/io500-opt.h b/include/io500-opt.h index 8a324e5..cb63920 100644 --- a/include/io500-opt.h +++ b/include/io500-opt.h @@ -26,7 +26,8 @@ typedef struct{ int rank; int mpi_size; - int io_buffers_on_gpu; /* are the I/O buffers to be allocated on a GPU */ + int allocateBufferDevice; /* where are the I/O buffers allocated and processed */ + int gpuDirect; /* useGPUDirect */ char * api; char * apiArgs; // for IOR and mdtest diff --git a/src/phase_ior_easy.c b/src/phase_ior_easy.c index 82fde10..54bbc83 100644 --- a/src/phase_ior_easy.c +++ b/src/phase_ior_easy.c @@ -50,9 +50,12 @@ void ior_easy_add_params(u_argv_t * argv, int useStatusFile){ for(int i=0; i < ior_easy_o.verbosity; i++){ u_argv_push(argv, "-v"); /* verbose */ } - if(opt.io_buffers_on_gpu){ + if(opt.allocateBufferDevice){ u_argv_push(argv, "-O"); - u_argv_push(argv, "allocateBufferOnGPU=1"); + u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice); + if(opt.gpuDirect){ + u_argv_push(argv, "--gpuDirect"); + } } u_argv_push(argv, "-C"); /* reorder tasks in constant order for read */ u_argv_push(argv, "-Q"); /* task per node offset */ diff --git a/src/phase_ior_hard.c b/src/phase_ior_hard.c index d052098..74425de 100644 --- a/src/phase_ior_hard.c +++ b/src/phase_ior_hard.c @@ -40,9 +40,12 @@ void ior_hard_add_params(u_argv_t * argv){ for(int i=0; i < ior_hard_o.verbosity; i++){ u_argv_push(argv, "-v"); /* verbose */ } - if(opt.io_buffers_on_gpu){ + if(opt.allocateBufferDevice){ u_argv_push(argv, "-O"); - u_argv_push(argv, "allocateBufferOnGPU=1"); + u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice); + if(opt.gpuDirect){ + u_argv_push(argv, "--gpuDirect"); + } } u_argv_push(argv, "-C"); /* reorder tasks in constant order for read */ u_argv_push(argv, "-Q"); /* task per node offset */ diff --git a/src/phase_ior_rnd1MB.c b/src/phase_ior_rnd1MB.c index a2fa1aa..4963eb3 100644 --- a/src/phase_ior_rnd1MB.c +++ b/src/phase_ior_rnd1MB.c @@ -38,9 +38,12 @@ void ior_rnd1MB_add_params(u_argv_t * argv){ for(int i=0; i < ior_rnd1MB_o.verbosity; i++){ u_argv_push(argv, "-v"); } - if(opt.io_buffers_on_gpu){ + if(opt.allocateBufferDevice){ u_argv_push(argv, "-O"); - u_argv_push(argv, "allocateBufferOnGPU=1"); + u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice); + if(opt.gpuDirect){ + u_argv_push(argv, "--gpuDirect"); + } } u_argv_push(argv, "-Q=1"); //u_argv_push(argv, "-F"); diff --git a/src/phase_ior_rnd4K.c b/src/phase_ior_rnd4K.c index 4584617..fc4f0b8 100644 --- a/src/phase_ior_rnd4K.c +++ b/src/phase_ior_rnd4K.c @@ -38,9 +38,12 @@ void ior_rnd4K_add_params(u_argv_t * argv){ for(int i=0; i < ior_rnd4K_o.verbosity; i++){ u_argv_push(argv, "-v"); } - if(opt.io_buffers_on_gpu){ + if(opt.allocateBufferDevice){ u_argv_push(argv, "-O"); - u_argv_push(argv, "allocateBufferOnGPU=1"); + u_argv_push_printf(argv, "allocateBufferOnGPU=%d", opt.allocateBufferDevice); + if(opt.gpuDirect){ + u_argv_push(argv, "--gpuDirect"); + } } u_argv_push(argv, "-Q=1"); //u_argv_push(argv, "-F"); diff --git a/src/phase_mdtest_easy.c b/src/phase_mdtest_easy.c index 4b9a424..6862a35 100644 --- a/src/phase_mdtest_easy.c +++ b/src/phase_mdtest_easy.c @@ -32,8 +32,11 @@ void mdtest_easy_add_params(u_argv_t * argv){ u_argv_push(argv, "./mdtest"); u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); - if(opt.io_buffers_on_gpu){ - u_argv_push(argv, "--allocateBufferOnGPU"); + if(opt.allocateBufferDevice){ + u_argv_push_printf(argv, "--allocateBufferOnGPU=%d", opt.allocateBufferDevice); + if(opt.gpuDirect){ + u_argv_push(argv, "--gpuDirect"); + } } u_argv_push(argv, "-n"); /* number of files per process */ u_argv_push_printf(argv, "%"PRIu64, d.g.files_per_proc); diff --git a/src/phase_mdtest_hard.c b/src/phase_mdtest_hard.c index 5c5244c..a1376dc 100644 --- a/src/phase_mdtest_hard.c +++ b/src/phase_mdtest_hard.c @@ -35,9 +35,13 @@ void mdtest_hard_add_params(u_argv_t * argv){ u_argv_push(argv, "./mdtest"); u_argv_push_printf(argv, "--dataPacketType=%s", opt.dataPacketType); - if(opt.io_buffers_on_gpu){ - u_argv_push(argv, "--allocateBufferOnGPU"); + if(opt.allocateBufferDevice){ + u_argv_push_printf(argv, "--allocateBufferOnGPU=%d", opt.allocateBufferDevice); + if(opt.gpuDirect){ + u_argv_push(argv, "--gpuDirect"); + } } + u_argv_push(argv, "-n"); /* number of files per process */ u_argv_push_printf(argv, "%"PRIu64, d.g.files_per_proc); diff --git a/src/phase_mdworkbench.c b/src/phase_mdworkbench.c index 3db8577..9fc646d 100644 --- a/src/phase_mdworkbench.c +++ b/src/phase_mdworkbench.c @@ -53,9 +53,12 @@ void mdworkbench_add_params(u_argv_t * argv, int is_create){ for(int i=0; i < mdworkbench_o.verbosity; i++){ u_argv_push(argv, "-v"); } - if(opt.io_buffers_on_gpu){ - u_argv_push(argv, "--allocateBufferOnGPU"); - } + if(opt.allocateBufferDevice){ + u_argv_push_printf(argv, "--allocateBufferOnGPU=%d", opt.allocateBufferDevice); + if(opt.gpuDirect){ + u_argv_push(argv, "--gpuDirect"); + } + } u_argv_push(argv, "--process-reports"); u_argv_push_default_if_set_api_options(argv, "-a", d->api, d->api); u_argv_push_printf(argv, "-o=%s/mdworkbench", opt.datadir); diff --git a/src/phase_opt.c b/src/phase_opt.c index ea8f176..a6884dd 100644 --- a/src/phase_opt.c +++ b/src/phase_opt.c @@ -10,7 +10,8 @@ static ini_option_t option[] = { {"api", "The general API for the tests (to create/delete the datadir, extra options will be passed to IOR/mdtest)", 0, INI_STRING, "POSIX", & opt.api}, {"drop-caches", "Purge the caches, this is useful for testing and needed for single node runs", 0, INI_BOOL, "FALSE", & opt.drop_caches}, {"drop-caches-cmd", "Cache purging command, invoked before each I/O phase", 0, INI_STRING, "sudo -n bash -c \"echo 3 > /proc/sys/vm/drop_caches\"", & opt.drop_caches_cmd}, - {"io-buffers-on-gpu", "Allocate the I/O buffers on the GPU", 0, INI_BOOL, "FALSE", & opt.io_buffers_on_gpu}, + {"allocateBufferDevice", "Allocate the I/O buffers on 0=CPU, 1=Managed Check CPU, 2=Managed Check GPU, 3=GPU", 0, INI_BOOL, "FALSE", & opt.allocateBufferDevice}, + {"gpuDirect", "Use GPUDirect only useful with Buffers > 0", 0, INI_BOOL, "FALSE", & opt.gpuDirect}, {"verbosity", "The verbosity level between 1 and 10", 0, INI_UINT, "1", & opt.verbosity}, {"scc", "Use the rules for the Student Cluster Competition", 0, INI_BOOL, "FALSE", & opt.scc}, {"dataPacketType", "Type of packet that will be created [timestamp|offset|incompressible|random]", 0, INI_STRING, "timestamp", & opt.dataPacketType}, From 4d6b43d672ac2ac148613dbd26b94db5fd75010d Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sun, 25 Dec 2022 12:05:55 +0100 Subject: [PATCH 20/21] Change allocation type --- src/phase_opt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/phase_opt.c b/src/phase_opt.c index a6884dd..fc018e4 100644 --- a/src/phase_opt.c +++ b/src/phase_opt.c @@ -10,7 +10,7 @@ static ini_option_t option[] = { {"api", "The general API for the tests (to create/delete the datadir, extra options will be passed to IOR/mdtest)", 0, INI_STRING, "POSIX", & opt.api}, {"drop-caches", "Purge the caches, this is useful for testing and needed for single node runs", 0, INI_BOOL, "FALSE", & opt.drop_caches}, {"drop-caches-cmd", "Cache purging command, invoked before each I/O phase", 0, INI_STRING, "sudo -n bash -c \"echo 3 > /proc/sys/vm/drop_caches\"", & opt.drop_caches_cmd}, - {"allocateBufferDevice", "Allocate the I/O buffers on 0=CPU, 1=Managed Check CPU, 2=Managed Check GPU, 3=GPU", 0, INI_BOOL, "FALSE", & opt.allocateBufferDevice}, + {"allocateBufferDevice", "Allocate the I/O buffers on 0=CPU, 1=Managed Check CPU, 2=Managed Check GPU, 3=GPU", 0, INI_INT, "0", & opt.allocateBufferDevice}, {"gpuDirect", "Use GPUDirect only useful with Buffers > 0", 0, INI_BOOL, "FALSE", & opt.gpuDirect}, {"verbosity", "The verbosity level between 1 and 10", 0, INI_UINT, "1", & opt.verbosity}, {"scc", "Use the rules for the Student Cluster Competition", 0, INI_BOOL, "FALSE", & opt.scc}, From 2cd466f8acae97ca0ca851651a11575a731f6fa8 Mon Sep 17 00:00:00 2001 From: "Julian M. Kunkel" Date: Sat, 5 Jul 2025 13:35:03 +0200 Subject: [PATCH 21/21] Bugfix output collection --- src/phase_concurrent.c | 41 ++++++++++++++++++++++++++++++++++------- src/phase_ior.c | 2 +- src/util.c | 2 +- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/phase_concurrent.c b/src/phase_concurrent.c index ea71a48..391818a 100644 --- a/src/phase_concurrent.c +++ b/src/phase_concurrent.c @@ -17,6 +17,7 @@ typedef struct{ int color; // benchmark that is run float ratio_write; float ratio_read; + int procs[3]; } opt_concurrent_bench; static opt_concurrent_bench o; @@ -47,10 +48,11 @@ static double run(void){ int workloads[] = {opt.mpi_size * d.ratio_write, opt.mpi_size * (d.ratio_write+d.ratio_read), opt.mpi_size}; int procs[] = {workloads[0], workloads[1] - workloads[0], workloads[2] - workloads[1]}; + memcpy(o.procs, procs, sizeof(int)*3); if(procs[0] + procs[1] == 0 || procs[0] + procs[2] == 0 || procs[1] + procs[2] == 0){ INVALID("The concurrent phase doesn't make sense with two benchmarks using 0 processes\n"); - return 0; + //return 0; } int color = (opt.rank < workloads[0]) ? 0 : (opt.rank < workloads[1] ? 1 : 2); @@ -75,6 +77,8 @@ static double run(void){ { opt_ior_easy d = ior_easy_o; ior_easy_add_params(argv[0], 0, 1); + u_argv_push(argv[0], "-o"); /* filename for output file */ + u_argv_push_printf(argv[0], "%s/ior-easy-concurrent/ior_file_easy", opt.datadir); u_argv_push(argv[0], "-w"); /* write file */ u_argv_push_printf(argv[0], "-D=%d", opt.stonewall); /* deadline for stonewall in seconds */ u_argv_push_printf(argv[0], "-O=minTimeDuration=%d", opt.stonewall); /* minimum runtime */ @@ -121,11 +125,6 @@ static double run(void){ double time = 0; if(color == 0){ // run ior easy write - if(ior_easy_o.filePerProc){ - char filename[2048]; - sprintf(filename, "ior-easy/ior_file_easy.%08d", opt.rank); - u_purge_file(filename); - } FILE * out = u_res_file_prep("concurrent-ior-easy-write", crank); score = ior_process_write(argv[0], out, & o.iorw, ccom); if(o.iorw){ @@ -196,12 +195,40 @@ static double run(void){ } +static void validate(void){ + u_create_datadir("ior-easy-concurrent"); +} + +static void cleanup(void){ + opt_concurrent_bench d = o; + + if (opt.dry_run || ! d.run) return; + + if(opt.rank == 0){ + if(! ior_easy_o.filePerProc){ + u_purge_file("ior-easy-concurrent/ior_file_easy"); + } + } + if(ior_easy_o.filePerProc){ + char filename[PATH_MAX]; + if(opt.rank < d.procs[0]){ + sprintf(filename, "ior-easy-concurrent/ior_file_easy.%08d", opt.rank); + u_purge_file(filename); + } + } + if(opt.rank == 0){ + u_purge_datadir("ior-easy-concurrent"); + } +} + + u_phase_t p_concurrent = { "concurrent", IO500_PHASE_WRITE | IO500_PHASE_FLAG_OPTIONAL, option, - NULL, + validate, run, 0, + .cleanup = cleanup, .group = IO500_SCORE_CONCURRENT, }; diff --git a/src/phase_ior.c b/src/phase_ior.c index 494eacb..ed30afd 100644 --- a/src/phase_ior.c +++ b/src/phase_ior.c @@ -40,7 +40,7 @@ double ior_process_read(u_argv_t * argv, FILE * out, IOR_point_t ** res_out, MPI *res_out = p; if(res->errors){ - INVALID("Errors (%d) occured during phase in IOR. This invalidates your run.\n", res->errors); + INVALID("Errors (%d) occurred during phase in IOR. This invalidates your run.\n", res->errors); } double tp = p->aggFileSizeForBW / p->time / GIBIBYTE; return tp; diff --git a/src/util.c b/src/util.c index 66c024d..caebfa3 100644 --- a/src/util.c +++ b/src/util.c @@ -237,7 +237,7 @@ void u_print_timestamp(FILE * out){ FILE * u_res_file_prep(char const * name, int rank){ FILE * out = stdout; - if(opt.rank == 0){ + if(rank == 0){ char fname[PATH_MAX]; sprintf(fname, "%s/%s.txt", opt.resdir, name); INFO_PAIR("result-file", "%s\n", fname);