From fc40b16c3d7a98dd47665ace419667293ad41b17 Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Wed, 27 Nov 2024 12:00:37 +0000 Subject: [PATCH 01/15] First attempt at regression test script. --- HMC/FTHMC2p1f.cc | 7 + HMC/FTHMC2p1f_3GeV.cc | 7 + HMC/HMC2p1f_3GeV.cc | 7 + tests/run_regression_test.py | 134 ++++++++++++++++++ .../Test_hmc_Sp_WilsonFundFermionGauge.cc | 6 +- ...hmc_Sp_WilsonFundFermionGauge_expected.txt | 1 + 6 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 tests/run_regression_test.py create mode 100644 tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt diff --git a/HMC/FTHMC2p1f.cc b/HMC/FTHMC2p1f.cc index 7d93d168f8..9432c464ad 100644 --- a/HMC/FTHMC2p1f.cc +++ b/HMC/FTHMC2p1f.cc @@ -25,13 +25,19 @@ directory *************************************************************************************/ /* END LEGAL */ #include +#if Nc == 3 #include #include +#endif using namespace Grid; int main(int argc, char **argv) { +#if Nc != 3 +#warning FTHMC2p1f will not work for Nc != 3 + std::cout << "This program will currently only work for Nc == 3." << std::endl; +#else std::cout << std::setprecision(12); Grid_init(&argc, &argv); @@ -220,6 +226,7 @@ int main(int argc, char **argv) TheHMC.Run(SmearingPolicy); // for smearing Grid_finalize(); +#endif } // main diff --git a/HMC/FTHMC2p1f_3GeV.cc b/HMC/FTHMC2p1f_3GeV.cc index a8aa67f8cf..2b7613a348 100644 --- a/HMC/FTHMC2p1f_3GeV.cc +++ b/HMC/FTHMC2p1f_3GeV.cc @@ -25,13 +25,19 @@ directory *************************************************************************************/ /* END LEGAL */ #include +#if Nc == 3 #include #include +#endif using namespace Grid; int main(int argc, char **argv) { +#if Nc != 3 +#warning FTHMC2p1f_3GeV will not work for Nc != 3 + std::cout << "This program will currently only work for Nc == 3." << std::endl; +#else std::cout << std::setprecision(12); Grid_init(&argc, &argv); @@ -220,6 +226,7 @@ int main(int argc, char **argv) TheHMC.Run(SmearingPolicy); // for smearing Grid_finalize(); +#endif } // main diff --git a/HMC/HMC2p1f_3GeV.cc b/HMC/HMC2p1f_3GeV.cc index 4bf088d7f0..ecc57006f0 100644 --- a/HMC/HMC2p1f_3GeV.cc +++ b/HMC/HMC2p1f_3GeV.cc @@ -25,13 +25,19 @@ directory *************************************************************************************/ /* END LEGAL */ #include +#if Nc == 3 #include #include +#endif using namespace Grid; int main(int argc, char **argv) { +#if Nc != 3 +#warning HMC2p1f_3GeV will not work for Nc != 3 + std::cout << "This program will currently only work for Nc == 3." << std::endl; +#else std::cout << std::setprecision(12); Grid_init(&argc, &argv); @@ -220,6 +226,7 @@ int main(int argc, char **argv) TheHMC.Run(SmearingPolicy); // for smearing Grid_finalize(); +#endif } // main diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py new file mode 100644 index 0000000000..c7ad036744 --- /dev/null +++ b/tests/run_regression_test.py @@ -0,0 +1,134 @@ +def read_expected(test_name="Test_hmc_Sp_WilsonFundFermionGauge", grid="8.8.8.8", mpi="1.1.1.1"): + """ + Read expected values from file. + + The file contains one or more entries of the following format: + + Eg. + 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c + """ + + with open(f"{test_name}_expected.txt") as file: + for line in file: + line_split = line.split() + if line_split[0] == grid and line_split[1] == mpi: + return float(line_split[2]), line_split[3], line_split[4] + + +def read_output(): + """ + Read test output and fish out values of interest. + """ + + checksum_rng = None + checksum_lat = None + plaquette = None + with open("output.txt", 'r') as file: + for line in file: + if "Written NERSC" in line: + subline = line.split('checksum ')[1] + if len(subline.split()) == 1: # this is the rng checksum line + checksum_rng = subline.strip() + elif len(subline.split()) == 3: # this is the lat checksum and plaquette value line + checksum_lat = subline.split()[0] + plaquette = float(subline.split()[2]) + else: + print("Picked wrong line...") + + if (checksum_rng is None) or (checksum_lat is None) or (plaquette is None): + print("Error reading values from output file. Make sure you compile the test with CPparams.saveInterval=1 in order to produce the required output.") + exit(1) + + return plaquette, checksum_rng, checksum_lat + + +def compare(actual, expected, what, stop=False): + """ + Compare actual with expected output, and output message if failed. + """ + + if actual != expected: + print(f"{what} comparison failed: actual={actual} , expected={expected}") + if stop: + exit(1) + else: + return False + return True + + + +if __name__ == '__main__': + import argparse + import subprocess + import os + + parser = argparse.ArgumentParser(description='Run end-to-end tests and compare results with expectations.') + parser.add_argument("test_name", help="File name of the test") + parser.add_argument("grid", help="Grid configuration") + parser.add_argument("mpi", help="MPI configuration") + parser.add_argument("-s", "--stop", action='store_true', help="Flag to stop testing when a test fails.") + args = parser.parse_args() + + expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected(args.test_name, args.grid, args.mpi) + + result = subprocess.run([f"./{args.test_name} --grid {args.grid} --mpi {args.mpi} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") + plaquette, checksum_rng, checksum_lat = read_output() + + print(f"Running {args.test_name}") + result = compare(plaquette, expected_plaquette, "plaquette", args.stop) + result = result and compare(checksum_rng, expected_checksum_rng, "Checksum RNG file ", args.stop) + result = result and compare(checksum_lat, expected_checksum_lat, "Checksum LAT file ", args.stop) + if result: + print("All tests passed!") + else: + print("Some tests failed...") + + os.remove("output.txt") + os.remove("ckpoint_rng.1") + os.remove("ckpoint_lat.1") + +#result = subprocess.run(["./Test_hmc_Sp_WilsonFundFermionGauge --grid 8.8.8.8 --mpi 1.1.1.1 --Thermalizations 0 --Trajectories 1 > output1.txt"], shell=True, encoding="text") + +# expected_value = 0.0256253844 +# checksum_rng = "922c392f" +# checksum_lat = "d1e4cc1c" + +# with open("output1.txt", 'r') as file: +# for line in file: +# # if "Plaquette" in line: +# # #print(line) +# # plaquette_value = float(line.split('] ')[1]) +# # #print(plaquette_value) +# # if plaquette_value == expected_value: +# # print("Success!") +# if "Written NERSC" in line: +# print(line) +# subline = line.split('checksum ')[1] +# if len(subline.split()) == 1: # this is the rng checksum line +# print(subline) +# if subline.strip() == checksum_rng: +# print("RNG file checksum success!") +# else: +# print("RNG file checksum failed!") +# elif len(subline.split()) == 3: # this is the lat checksum and plaquette value line +# print(subline) +# checksum_value = subline.split()[0] +# plaquette_value = float(subline.split()[2]) +# print(checksum_value, plaquette_value) +# if checksum_value == checksum_lat: +# print("LAT file checksum success!") +# else: +# print("LAT file checksum failed!") +# if plaquette_value == expected_value: +# print("Plaquette value success!") +# else: +# print("Plaquette value failed!") +# else: +# print("Picked wrong line...") + + +#loc1 = result.find("Plaquette") +#print(loc1) +#loc2 = result.find("Smeared") +#print(loc2) +#print(result[loc1,loc2]) diff --git a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc index d302911df1..357c606540 100644 --- a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc +++ b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc @@ -22,7 +22,7 @@ int main(int argc, char **argv) { CheckpointerParameters CPparams; CPparams.config_prefix = "ckpoint_lat"; CPparams.rng_prefix = "ckpoint_rng"; - CPparams.saveInterval = 100; + CPparams.saveInterval = 1; CPparams.format = "IEEE64BIG"; TheHMC.Resources.LoadNerscCheckpointer(CPparams); @@ -64,8 +64,8 @@ int main(int argc, char **argv) { TheHMC.TheAction.push_back(Level1); TheHMC.TheAction.push_back(Level2); - TheHMC.Parameters.MD.MDsteps = 36; - TheHMC.Parameters.MD.trajL = 1.0; + TheHMC.Parameters.MD.MDsteps = 2; + TheHMC.Parameters.MD.trajL = 0.1; TheHMC.ReadCommandLine(argc, argv); TheHMC.Run(); diff --git a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt new file mode 100644 index 0000000000..7edc6fa0a9 --- /dev/null +++ b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt @@ -0,0 +1 @@ +8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c From fc8822a015400dab7de48c2b69b70df19c635381 Mon Sep 17 00:00:00 2001 From: Mashy Green Date: Tue, 3 Dec 2024 15:25:07 +0000 Subject: [PATCH 02/15] Adding expected values for 2 additional SU3 tests --- tests/hmc/Test_hmc_IwasakiGauge_expected.txt | 1 + tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/hmc/Test_hmc_IwasakiGauge_expected.txt create mode 100644 tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt diff --git a/tests/hmc/Test_hmc_IwasakiGauge_expected.txt b/tests/hmc/Test_hmc_IwasakiGauge_expected.txt new file mode 100644 index 0000000000..9391cc8c3d --- /dev/null +++ b/tests/hmc/Test_hmc_IwasakiGauge_expected.txt @@ -0,0 +1 @@ +8.8.8.8 1.1.1.1 0.269298793 633bf471 3a22ad20 diff --git a/tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt b/tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt new file mode 100644 index 0000000000..6b53130bfa --- /dev/null +++ b/tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt @@ -0,0 +1 @@ +8.8.8.8 1.1.1.1 0.254950222 7f25d41 9d40279f From 8531bf161a09cfdc2093a37a508e2a398de7dfd6 Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Wed, 18 Dec 2024 12:23:47 +0000 Subject: [PATCH 03/15] Apply suggestions from code review Co-authored-by: Mashy Green --- tests/run_regression_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index c7ad036744..72decc6ca0 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -71,7 +71,7 @@ def compare(actual, expected, what, stop=False): expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected(args.test_name, args.grid, args.mpi) - result = subprocess.run([f"./{args.test_name} --grid {args.grid} --mpi {args.mpi} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") + result = subprocess.run([f"./{args.test_name} --grid {args.grid} --mpi {args.mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") plaquette, checksum_rng, checksum_lat = read_output() print(f"Running {args.test_name}") @@ -80,10 +80,10 @@ def compare(actual, expected, what, stop=False): result = result and compare(checksum_lat, expected_checksum_lat, "Checksum LAT file ", args.stop) if result: print("All tests passed!") + os.remove("output.txt") else: print("Some tests failed...") - os.remove("output.txt") os.remove("ckpoint_rng.1") os.remove("ckpoint_lat.1") From 935201587c04f9df558850eda2a9ccda95b967fb Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Thu, 19 Dec 2024 10:04:17 +0000 Subject: [PATCH 04/15] Initial port to pytest - messy! --- tests/conftest.py | 36 ++++++++++++ tests/run_regression_test.py | 111 +++++++++++------------------------ 2 files changed, 69 insertions(+), 78 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..f996409419 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,36 @@ +import pytest + +def pytest_addoption(parser): + parser.addoption( + "--test_name", action="store", help="File name of the test" + ) + parser.addoption( + "--grid", action="store", help="Grid configuration" + ) + parser.addoption( + "--mpi", action="store", help="MPI configuration" + ) + +@pytest.fixture +def test_name(request): + return request.config.getoption("--test_name") + +@pytest.fixture +def grid(request): + return request.config.getoption("--grid") + +@pytest.fixture +def mpi(request): + return request.config.getoption("--mpi") + +@pytest.fixture(scope="function", autouse=True) +def cleanup_files(): + import os + + print("Before!") + yield + print(f"After!") + os.remove("output.txt") + + os.remove("ckpoint_rng.1") + os.remove("ckpoint_lat.1") \ No newline at end of file diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index 72decc6ca0..e432279691 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -1,3 +1,5 @@ +import pytest + def read_expected(test_name="Test_hmc_Sp_WilsonFundFermionGauge", grid="8.8.8.8", mpi="1.1.1.1"): """ Read expected values from file. @@ -42,25 +44,9 @@ def read_output(): return plaquette, checksum_rng, checksum_lat -def compare(actual, expected, what, stop=False): - """ - Compare actual with expected output, and output message if failed. - """ - - if actual != expected: - print(f"{what} comparison failed: actual={actual} , expected={expected}") - if stop: - exit(1) - else: - return False - return True - - - -if __name__ == '__main__': +@pytest.fixture +def cl_arguments(): import argparse - import subprocess - import os parser = argparse.ArgumentParser(description='Run end-to-end tests and compare results with expectations.') parser.add_argument("test_name", help="File name of the test") @@ -69,66 +55,35 @@ def compare(actual, expected, what, stop=False): parser.add_argument("-s", "--stop", action='store_true', help="Flag to stop testing when a test fails.") args = parser.parse_args() - expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected(args.test_name, args.grid, args.mpi) + return args + + +#def test_outputs(cl_arguments): +def test_outputs(test_name, grid, mpi): + import subprocess + import os + + #test_name = cl_arguments.test_name + #expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected(test_name, cl_arguments.grid, cl_arguments.mpi) + expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected(test_name, grid, mpi) - result = subprocess.run([f"./{args.test_name} --grid {args.grid} --mpi {args.mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") + #result = subprocess.run([f"./{test_name} --grid {cl_arguments.grid} --mpi {cl_arguments.mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") + #result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") + result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") plaquette, checksum_rng, checksum_lat = read_output() - print(f"Running {args.test_name}") - result = compare(plaquette, expected_plaquette, "plaquette", args.stop) - result = result and compare(checksum_rng, expected_checksum_rng, "Checksum RNG file ", args.stop) - result = result and compare(checksum_lat, expected_checksum_lat, "Checksum LAT file ", args.stop) - if result: - print("All tests passed!") - os.remove("output.txt") - else: - print("Some tests failed...") - - os.remove("ckpoint_rng.1") - os.remove("ckpoint_lat.1") - -#result = subprocess.run(["./Test_hmc_Sp_WilsonFundFermionGauge --grid 8.8.8.8 --mpi 1.1.1.1 --Thermalizations 0 --Trajectories 1 > output1.txt"], shell=True, encoding="text") - -# expected_value = 0.0256253844 -# checksum_rng = "922c392f" -# checksum_lat = "d1e4cc1c" - -# with open("output1.txt", 'r') as file: -# for line in file: -# # if "Plaquette" in line: -# # #print(line) -# # plaquette_value = float(line.split('] ')[1]) -# # #print(plaquette_value) -# # if plaquette_value == expected_value: -# # print("Success!") -# if "Written NERSC" in line: -# print(line) -# subline = line.split('checksum ')[1] -# if len(subline.split()) == 1: # this is the rng checksum line -# print(subline) -# if subline.strip() == checksum_rng: -# print("RNG file checksum success!") -# else: -# print("RNG file checksum failed!") -# elif len(subline.split()) == 3: # this is the lat checksum and plaquette value line -# print(subline) -# checksum_value = subline.split()[0] -# plaquette_value = float(subline.split()[2]) -# print(checksum_value, plaquette_value) -# if checksum_value == checksum_lat: -# print("LAT file checksum success!") -# else: -# print("LAT file checksum failed!") -# if plaquette_value == expected_value: -# print("Plaquette value success!") -# else: -# print("Plaquette value failed!") -# else: -# print("Picked wrong line...") - - -#loc1 = result.find("Plaquette") -#print(loc1) -#loc2 = result.find("Smeared") -#print(loc2) -#print(result[loc1,loc2]) + print(f"Running {test_name}") + assert plaquette == expected_plaquette + assert checksum_rng == expected_checksum_rng + assert checksum_lat == expected_checksum_lat + #result = compare(plaquette, expected_plaquette, "plaquette", args.stop) + #result = result and compare(checksum_rng, expected_checksum_rng, "Checksum RNG file ", args.stop) + #result = result and compare(checksum_lat, expected_checksum_lat, "Checksum LAT file ", args.stop) + # if result: + # print("All tests passed!") + # os.remove("output.txt") + # else: + # print("Some tests failed...") + + # os.remove("ckpoint_rng.1") + # os.remove("ckpoint_lat.1") From 7de81af2104eba1eb2657b6cf743000f5e8c2138 Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Thu, 19 Dec 2024 12:59:24 +0000 Subject: [PATCH 05/15] Clean pytest implementation. --- tests/conftest.py | 14 ++++++------ tests/run_regression_test.py | 44 +++++++++--------------------------- 2 files changed, 18 insertions(+), 40 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f996409419..104c947bb8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,14 +23,14 @@ def grid(request): def mpi(request): return request.config.getoption("--mpi") -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture def cleanup_files(): import os - print("Before!") - yield - print(f"After!") - os.remove("output.txt") + def _cleanup(failed=True): + if not failed: + if os.path.exists("output.txt"): os.remove("output.txt") + if os.path.exists("ckpoint_rng.1"): os.remove("ckpoint_rng.1") + if os.path.exists("ckpoint_lat.1"): os.remove("ckpoint_lat.1") - os.remove("ckpoint_rng.1") - os.remove("ckpoint_lat.1") \ No newline at end of file + return _cleanup \ No newline at end of file diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index e432279691..32fa6304cb 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -1,6 +1,7 @@ import pytest +import pytest_check as check -def read_expected(test_name="Test_hmc_Sp_WilsonFundFermionGauge", grid="8.8.8.8", mpi="1.1.1.1"): +def read_expected(test_name, grid="8.8.8.8", mpi="1.1.1.1"): """ Read expected values from file. @@ -44,46 +45,23 @@ def read_output(): return plaquette, checksum_rng, checksum_lat -@pytest.fixture -def cl_arguments(): - import argparse - - parser = argparse.ArgumentParser(description='Run end-to-end tests and compare results with expectations.') - parser.add_argument("test_name", help="File name of the test") - parser.add_argument("grid", help="Grid configuration") - parser.add_argument("mpi", help="MPI configuration") - parser.add_argument("-s", "--stop", action='store_true', help="Flag to stop testing when a test fails.") - args = parser.parse_args() - - return args - - -#def test_outputs(cl_arguments): -def test_outputs(test_name, grid, mpi): +def test_outputs(test_name, grid, mpi, cleanup_files): import subprocess import os - #test_name = cl_arguments.test_name - #expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected(test_name, cl_arguments.grid, cl_arguments.mpi) expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected(test_name, grid, mpi) - #result = subprocess.run([f"./{test_name} --grid {cl_arguments.grid} --mpi {cl_arguments.mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") #result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") plaquette, checksum_rng, checksum_lat = read_output() print(f"Running {test_name}") - assert plaquette == expected_plaquette - assert checksum_rng == expected_checksum_rng - assert checksum_lat == expected_checksum_lat - #result = compare(plaquette, expected_plaquette, "plaquette", args.stop) - #result = result and compare(checksum_rng, expected_checksum_rng, "Checksum RNG file ", args.stop) - #result = result and compare(checksum_lat, expected_checksum_lat, "Checksum LAT file ", args.stop) - # if result: - # print("All tests passed!") - # os.remove("output.txt") - # else: - # print("Some tests failed...") + # This manual check of each condition doesn't have to happen for pytest-check + # version 1.2.0 and later. We can use any_failures() instead. + failed = False + if not check.equal(plaquette, expected_plaquette, msg="Plaquette value comparison failed") : failed = True + if not check.equal(checksum_lat, expected_checksum_lat, msg="LAT file checksum comparison failed") : failed = True + if not check.equal(checksum_rng, expected_checksum_rng, msg="RND file checksum comparison failed") : failed = True + + cleanup_files(failed) - # os.remove("ckpoint_rng.1") - # os.remove("ckpoint_lat.1") From ec81f62a3cfb5fe27f629cad43fe8cd418425ebe Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Wed, 29 Jan 2025 11:39:34 +0000 Subject: [PATCH 06/15] Add function to read expected values by line. SOme improvements in error checking and reporting. --- tests/run_regression_test.py | 50 ++++++++++++++----- ...hmc_Sp_WilsonFundFermionGauge_expected.txt | 2 +- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index 32fa6304cb..54612f6070 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -1,21 +1,47 @@ import pytest import pytest_check as check -def read_expected(test_name, grid="8.8.8.8", mpi="1.1.1.1"): +def read_expected_values(test_name, grid="8.8.8.8", mpi="1.1.1.1"): """ Read expected values from file. The file contains one or more entries of the following format: - + Eg. - 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c + 1 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c + + This function will return the values only for the correct grid + and mpi arguments. + """ + + with open(f"{test_name}_expected.txt") as file: + for line in file: + line_split = line.split() + if line_split[1] == grid and line_split[2] == mpi: + return float(line_split[3]), line_split[4], line_split[5] + + return None, None, None + +def read_expected_values_line(test_name, line_number=1): + """ + Read expected values from file. + + The file contains one or more entries of the following format: + + Eg. + 1 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c + + This function will return the values only for the requested line_number. """ with open(f"{test_name}_expected.txt") as file: for line in file: line_split = line.split() - if line_split[0] == grid and line_split[1] == mpi: - return float(line_split[2]), line_split[3], line_split[4] + if line_split[0] == line_number: + print(f"Reading reference values for grid={line_split[1]}, mpi={line_split[2]}") + return float(line_split[3]), line_split[4], line_split[5] + + return None, None, None def read_output(): @@ -38,10 +64,6 @@ def read_output(): else: print("Picked wrong line...") - if (checksum_rng is None) or (checksum_lat is None) or (plaquette is None): - print("Error reading values from output file. Make sure you compile the test with CPparams.saveInterval=1 in order to produce the required output.") - exit(1) - return plaquette, checksum_rng, checksum_lat @@ -49,12 +71,16 @@ def test_outputs(test_name, grid, mpi, cleanup_files): import subprocess import os - expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected(test_name, grid, mpi) + expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected_values(test_name, grid, mpi) + if (expected_plaquette is None) or (expected_checksum_rng is None) or (expected_checksum_lat is None): + pytest.fail(f"No appropriate reference values found, check {test_name}_expected.txt") #result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") - result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") + #result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") plaquette, checksum_rng, checksum_lat = read_output() - + if (checksum_rng is None) or (checksum_lat is None) or (plaquette is None): + pytest.fail("Error reading values from output file. Make sure you compile the test with CPparams.saveInterval=1 in order to produce the required output.") + print(f"Running {test_name}") # This manual check of each condition doesn't have to happen for pytest-check # version 1.2.0 and later. We can use any_failures() instead. diff --git a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt index 7edc6fa0a9..79afcf0603 100644 --- a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt +++ b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt @@ -1 +1 @@ -8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c +1 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c \ No newline at end of file From 57a004956397cb5bd3b1ddb4ba6921a1e49791a8 Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Wed, 29 Jan 2025 11:51:16 +0000 Subject: [PATCH 07/15] Read expected values from file by line. --- tests/conftest.py | 7 +++++++ tests/run_regression_test.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 104c947bb8..4f232d9c0d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,9 @@ def pytest_addoption(parser): parser.addoption( "--mpi", action="store", help="MPI configuration" ) + parser.addoption( + "--expected_line", action="store", help="Line number to read off the expected values file" + ) @pytest.fixture def test_name(request): @@ -23,6 +26,10 @@ def grid(request): def mpi(request): return request.config.getoption("--mpi") +@pytest.fixture +def expected_line(request): + return request.config.getoption("--expected_line") + @pytest.fixture def cleanup_files(): import os diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index 54612f6070..af42f1ca1c 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -67,11 +67,11 @@ def read_output(): return plaquette, checksum_rng, checksum_lat -def test_outputs(test_name, grid, mpi, cleanup_files): +def test_outputs(test_name, grid, mpi, expected_line, cleanup_files): import subprocess import os - expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected_values(test_name, grid, mpi) + expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected_values_line(test_name, expected_line) if (expected_plaquette is None) or (expected_checksum_rng is None) or (expected_checksum_lat is None): pytest.fail(f"No appropriate reference values found, check {test_name}_expected.txt") From a15688c62edc76f9145edb994aa9267e330934a8 Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Wed, 29 Jan 2025 15:49:45 +0000 Subject: [PATCH 08/15] Read test parameters as well from the file, remove from command line. --- tests/run_regression_test.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index af42f1ca1c..520bd96080 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -24,24 +24,27 @@ def read_expected_values(test_name, grid="8.8.8.8", mpi="1.1.1.1"): def read_expected_values_line(test_name, line_number=1): """ - Read expected values from file. + Read test parameters and expected values from file. The file contains one or more entries of the following format: Eg. 1 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c - This function will return the values only for the requested line_number. + This function will return the test parameters and expected values + only for the requested line_number. """ + test_parameters = {} with open(f"{test_name}_expected.txt") as file: for line in file: line_split = line.split() if line_split[0] == line_number: print(f"Reading reference values for grid={line_split[1]}, mpi={line_split[2]}") - return float(line_split[3]), line_split[4], line_split[5] + test_parameters = {'grid': line_split[1], 'mpi': line_split[2]} + return test_parameters, float(line_split[3]), line_split[4], line_split[5] - return None, None, None + return test_parameters, None, None, None def read_output(): @@ -67,21 +70,21 @@ def read_output(): return plaquette, checksum_rng, checksum_lat -def test_outputs(test_name, grid, mpi, expected_line, cleanup_files): +def test_outputs(test_name, expected_line, cleanup_files): import subprocess import os - expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected_values_line(test_name, expected_line) - if (expected_plaquette is None) or (expected_checksum_rng is None) or (expected_checksum_lat is None): + test_parameters, expected_plaquette, expected_checksum_rng, expected_checksum_lat = read_expected_values_line(test_name, expected_line) + if (not test_parameters) or (expected_plaquette is None) or (expected_checksum_rng is None) or (expected_checksum_lat is None): pytest.fail(f"No appropriate reference values found, check {test_name}_expected.txt") + print(f"Running {test_name} for test parameters: ", test_parameters) #result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") - #result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") + result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") plaquette, checksum_rng, checksum_lat = read_output() if (checksum_rng is None) or (checksum_lat is None) or (plaquette is None): pytest.fail("Error reading values from output file. Make sure you compile the test with CPparams.saveInterval=1 in order to produce the required output.") - print(f"Running {test_name}") # This manual check of each condition doesn't have to happen for pytest-check # version 1.2.0 and later. We can use any_failures() instead. failed = False From bed8c17a7b7fccb50a281dce6130ae3dadd9e54c Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Wed, 29 Jan 2025 15:51:28 +0000 Subject: [PATCH 09/15] Link over needed files for tests with autoconf. Future test reference value files added, will have to be linked over explicitly here as well. autotools does not do wildcards. --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 8e8d67af77..9f7f9921cd 100644 --- a/configure.ac +++ b/configure.ac @@ -860,6 +860,7 @@ AC_CONFIG_FILES(tests/testu01/Makefile) AC_CONFIG_FILES(tests/sp2n/Makefile) AC_CONFIG_FILES(benchmarks/Makefile) AC_CONFIG_FILES(examples/Makefile) +AC_CONFIG_LINKS([tests/run_regression_test.py:tests/run_regression_test.py tests/conftest.py:tests/conftest.py tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt:tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt]) AC_OUTPUT echo "" From 565a0514245d09cdaf311e6247fa303ef5637cbe Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Fri, 31 Jan 2025 14:23:52 +0000 Subject: [PATCH 10/15] Add nthreads as test parameter --- tests/run_regression_test.py | 40 ++++++------------- ...hmc_Sp_WilsonFundFermionGauge_expected.txt | 3 +- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index 520bd96080..c2c873d53b 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -1,35 +1,14 @@ import pytest import pytest_check as check -def read_expected_values(test_name, grid="8.8.8.8", mpi="1.1.1.1"): - """ - Read expected values from file. - - The file contains one or more entries of the following format: - - Eg. - 1 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c - - This function will return the values only for the correct grid - and mpi arguments. - """ - - with open(f"{test_name}_expected.txt") as file: - for line in file: - line_split = line.split() - if line_split[1] == grid and line_split[2] == mpi: - return float(line_split[3]), line_split[4], line_split[5] - - return None, None, None - def read_expected_values_line(test_name, line_number=1): """ Read test parameters and expected values from file. The file contains one or more entries of the following format: - + Eg. - 1 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c + 1 8.8.8.8 1.1.1.1 1 0.0256253844 922c392f d1e4cc1c This function will return the test parameters and expected values only for the requested line_number. @@ -40,9 +19,11 @@ def read_expected_values_line(test_name, line_number=1): for line in file: line_split = line.split() if line_split[0] == line_number: - print(f"Reading reference values for grid={line_split[1]}, mpi={line_split[2]}") - test_parameters = {'grid': line_split[1], 'mpi': line_split[2]} - return test_parameters, float(line_split[3]), line_split[4], line_split[5] + test_parameters['grid'] = line_split[1] + test_parameters['mpi'] = line_split[2] + test_parameters['nthreads'] = line_split[3] + print("Reading reference values for ", test_parameters) + return test_parameters, float(line_split[4]), line_split[5], line_split[6] return test_parameters, None, None, None @@ -79,8 +60,11 @@ def test_outputs(test_name, expected_line, cleanup_files): pytest.fail(f"No appropriate reference values found, check {test_name}_expected.txt") print(f"Running {test_name} for test parameters: ", test_parameters) - #result = subprocess.run([f"./{test_name} --grid {grid} --mpi {mpi} --Thermalizations 0 --Trajectories 1 --threads 1 > output.txt"], shell=True, encoding="text") - result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") + if test_parameters['nthreads'] == '0': + result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") + else: + print("RUnning nthreads .ne. 0") + result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 --threads {test_parameters['nthreads']} > output.txt"], shell=True, encoding="text") plaquette, checksum_rng, checksum_lat = read_output() if (checksum_rng is None) or (checksum_lat is None) or (plaquette is None): pytest.fail("Error reading values from output file. Make sure you compile the test with CPparams.saveInterval=1 in order to produce the required output.") diff --git a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt index 79afcf0603..cb63e72dad 100644 --- a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt +++ b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt @@ -1 +1,2 @@ -1 8.8.8.8 1.1.1.1 0.0256253844 922c392f d1e4cc1c \ No newline at end of file +1 8.8.8.8 1.1.1.1 0 0.0256253844 922c392f d1e4cc1c +2 8.8.8.8 1.1.1.1 1 0.0256253844 922c392f d1e4f305 \ No newline at end of file From 85e890650488544a18498ff6d801b46c28b2e00c Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Fri, 31 Jan 2025 16:51:57 +0000 Subject: [PATCH 11/15] Add the rest of the requested test parameters. --- tests/run_regression_test.py | 40 +++++++++++++++---- ...hmc_Sp_WilsonFundFermionGauge_expected.txt | 5 ++- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index c2c873d53b..388ad0e27f 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -6,9 +6,9 @@ def read_expected_values_line(test_name, line_number=1): Read test parameters and expected values from file. The file contains one or more entries of the following format: - + Eg. - 1 8.8.8.8 1.1.1.1 1 0.0256253844 922c392f d1e4cc1c + 1 8.8.8.8 1.1.1.1 1 2 0.1 CPU 0.0256253844 922c392f d1e4f305 This function will return the test parameters and expected values only for the requested line_number. @@ -22,23 +22,45 @@ def read_expected_values_line(test_name, line_number=1): test_parameters['grid'] = line_split[1] test_parameters['mpi'] = line_split[2] test_parameters['nthreads'] = line_split[3] + test_parameters['MDsteps'] = line_split[4] + test_parameters['trajL'] = line_split[5] + test_parameters['CPU|GPU'] = line_split[6] print("Reading reference values for ", test_parameters) - return test_parameters, float(line_split[4]), line_split[5], line_split[6] + return test_parameters, float(line_split[7]), line_split[8], line_split[9] return test_parameters, None, None, None -def read_output(): +def read_output(test_parameters): """ Read test output and fish out values of interest. """ + MDsteps = None + trajL = None + CPUvsGPU = 'CPU' + checked_CPUvsGPU = False checksum_rng = None checksum_lat = None plaquette = None + with open("output.txt", 'r') as file: for line in file: - if "Written NERSC" in line: + # Check that the test was run with the expected parameters + if not checked_CPUvsGPU: + if "cuda" in line: + CPUvsGPU = 'GPU' + checked_CPUvsGPU = True + if "Number of MD steps" in line: + MDsteps = line.split(' : ')[4].strip() + if MDsteps != test_parameters['MDsteps']: + pytest.fail(f"Test was run with MDsteps={MDsteps} instead of {test_parameters['MDsteps']}") + elif "Trajectory length" in line: + trajL = line.split(' : ')[4].strip() + if trajL != test_parameters['trajL']: + pytest.fail(f"Test was run with trajL={trajL} instead of {test_parameters['trajL']}") + # Read the values to test + elif "Written NERSC" in line: subline = line.split('checksum ')[1] if len(subline.split()) == 1: # this is the rng checksum line checksum_rng = subline.strip() @@ -48,6 +70,11 @@ def read_output(): else: print("Picked wrong line...") + if CPUvsGPU != test_parameters['CPU|GPU']: + pytest.fail(f"Test was run with {CPUvsGPU} instead of {test_parameters['CPU|GPU']}") + if (MDsteps is None) or (trajL is None): + pytest.fail("Could not verify test parameters MDsteps and/or trajL against test output.") + return plaquette, checksum_rng, checksum_lat @@ -63,9 +90,8 @@ def test_outputs(test_name, expected_line, cleanup_files): if test_parameters['nthreads'] == '0': result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") else: - print("RUnning nthreads .ne. 0") result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 --threads {test_parameters['nthreads']} > output.txt"], shell=True, encoding="text") - plaquette, checksum_rng, checksum_lat = read_output() + plaquette, checksum_rng, checksum_lat = read_output(test_parameters) if (checksum_rng is None) or (checksum_lat is None) or (plaquette is None): pytest.fail("Error reading values from output file. Make sure you compile the test with CPparams.saveInterval=1 in order to produce the required output.") diff --git a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt index cb63e72dad..a3995cf96e 100644 --- a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt +++ b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt @@ -1,2 +1,3 @@ -1 8.8.8.8 1.1.1.1 0 0.0256253844 922c392f d1e4cc1c -2 8.8.8.8 1.1.1.1 1 0.0256253844 922c392f d1e4f305 \ No newline at end of file +1 8.8.8.8 1.1.1.1 0 36 1.0 CPU 0.281507042 922c392f 716ec776 +2 8.8.8.8 1.1.1.1 0 2 0.1 CPU 0.0256253844 922c392f d1e4cc1c +3 8.8.8.8 1.1.1.1 1 2 0.1 CPU 0.0256253844 922c392f d1e4f305 \ No newline at end of file From 77d51194a56945663672237084f59210e5ed2ab9 Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Fri, 31 Jan 2025 16:55:30 +0000 Subject: [PATCH 12/15] Keep MDsteps and trajL in the test as they were Co-authored-by: Mashy Green --- tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc index 357c606540..83768f452d 100644 --- a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc +++ b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge.cc @@ -64,8 +64,8 @@ int main(int argc, char **argv) { TheHMC.TheAction.push_back(Level1); TheHMC.TheAction.push_back(Level2); - TheHMC.Parameters.MD.MDsteps = 2; - TheHMC.Parameters.MD.trajL = 0.1; + TheHMC.Parameters.MD.MDsteps = 36; + TheHMC.Parameters.MD.trajL = 1.0; TheHMC.ReadCommandLine(argc, argv); TheHMC.Run(); From 187b5497074616e93d601f73b83757fafdba39f0 Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Thu, 13 Feb 2025 16:49:12 +0000 Subject: [PATCH 13/15] Add documentation to run_regression_tests script. Various debugging. --- tests/run_regression_test.py | 48 ++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index 388ad0e27f..935d85c45e 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -1,3 +1,37 @@ +""" +Steering script to run end-to-end regression tests. + +This script: +1. Reads the parameters to run the test with and the expected values + of certain outputs to compare against, from a file. +2. Runs the requested test with the parameters read. + The executable must exist in the local directory, it is not recompiled. +3. Picks the output values from the output of the test. +4. Compares output values with expected. +5. Deletes the output files only if the test was successful. + +Dependencies: pytest, pytest-check + +The script should be run from the desired build/tests/, +which should contain +1. the test executable and +2. a file containing combinations of test parameters and the output values + they are expected to produce, named _expected.txt +When adding new tests, please add the new _expected.txt file +to the repo and ammend configure.ac to link it to the build location +with AC_CONFIG_LINKS. + +To run and produce nice reporting: +pytest ../run_regression_test.py -rP --test_name= --expected_line= +where + is the name of the test executable + is the number of the line in the _expected.txt file to read the +test parameters and expected output values from + +For example, from build/tests/sp2n: +pytest ../run_regression_test.py -rP --test_name=Test_hmc_Sp_WilsonFundFermionGauge --expected_line=2 +""" + import pytest import pytest_check as check @@ -18,7 +52,7 @@ def read_expected_values_line(test_name, line_number=1): with open(f"{test_name}_expected.txt") as file: for line in file: line_split = line.split() - if line_split[0] == line_number: + if line_split and line_split[0] == line_number: test_parameters['grid'] = line_split[1] test_parameters['mpi'] = line_split[2] test_parameters['nthreads'] = line_split[3] @@ -52,13 +86,13 @@ def read_output(test_parameters): CPUvsGPU = 'GPU' checked_CPUvsGPU = True if "Number of MD steps" in line: - MDsteps = line.split(' : ')[4].strip() - if MDsteps != test_parameters['MDsteps']: - pytest.fail(f"Test was run with MDsteps={MDsteps} instead of {test_parameters['MDsteps']}") + MDsteps = int(line.split(' : ')[4].strip()) + if MDsteps != int(test_parameters['MDsteps']): + pytest.fail(f"Test was run with MDsteps={MDsteps} instead of {test_parameters['MDsteps']}. You need to modify the test source code and recompile.") elif "Trajectory length" in line: - trajL = line.split(' : ')[4].strip() - if trajL != test_parameters['trajL']: - pytest.fail(f"Test was run with trajL={trajL} instead of {test_parameters['trajL']}") + trajL = float(line.split(' : ')[4].strip()) + if trajL != float(test_parameters['trajL']): + pytest.fail(f"Test was run with trajL={trajL} instead of {test_parameters['trajL']}. You need to modify the test source code and recompile.") # Read the values to test elif "Written NERSC" in line: subline = line.split('checksum ')[1] From dc184a4b22ef407d227bad52915f6f3ef685d6ff Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Thu, 13 Feb 2025 16:51:06 +0000 Subject: [PATCH 14/15] Add another test case to Test_hmc_Sp_WilsonFundFermionGauge --- tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt index a3995cf96e..26b0d5ac9c 100644 --- a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt +++ b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt @@ -1,3 +1,4 @@ 1 8.8.8.8 1.1.1.1 0 36 1.0 CPU 0.281507042 922c392f 716ec776 -2 8.8.8.8 1.1.1.1 0 2 0.1 CPU 0.0256253844 922c392f d1e4cc1c -3 8.8.8.8 1.1.1.1 1 2 0.1 CPU 0.0256253844 922c392f d1e4f305 \ No newline at end of file +2 8.8.8.8 1.1.1.1 1 36 1.0 CPU 0.281507042 922c392f 6a8bb5f6 +3 8.8.8.8 1.1.1.1 0 2 0.1 CPU 0.0256253844 922c392f d1e4cc1c +4 8.8.8.8 1.1.1.1 1 2 0.1 CPU 0.0256253844 922c392f d1e4f305 \ No newline at end of file From c70335cd237d2a0c38ed436616a3576242ad1f7d Mon Sep 17 00:00:00 2001 From: Ilektra Christidi Date: Mon, 24 Feb 2025 12:14:25 +0000 Subject: [PATCH 15/15] Add the hmc tests to configure.ac. Add some more test parameters and expected values - in progress... Modify saveInterval in hmc tests to produce outputs. Fix number of threads to 1 if not specified. --- configure.ac | 7 ++++++- tests/hmc/Test_hmc_IwasakiGauge.cc | 2 +- tests/hmc/Test_hmc_IwasakiGauge_expected.txt | 4 +++- tests/hmc/Test_hmc_WilsonFermionGauge.cc | 2 +- tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt | 4 +++- tests/run_regression_test.py | 6 +++--- tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt | 4 ++-- 7 files changed, 19 insertions(+), 10 deletions(-) diff --git a/configure.ac b/configure.ac index 9f7f9921cd..ff0ce143f8 100644 --- a/configure.ac +++ b/configure.ac @@ -860,7 +860,12 @@ AC_CONFIG_FILES(tests/testu01/Makefile) AC_CONFIG_FILES(tests/sp2n/Makefile) AC_CONFIG_FILES(benchmarks/Makefile) AC_CONFIG_FILES(examples/Makefile) -AC_CONFIG_LINKS([tests/run_regression_test.py:tests/run_regression_test.py tests/conftest.py:tests/conftest.py tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt:tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt]) +AC_CONFIG_LINKS(tests/run_regression_test.py:tests/run_regression_test.py) +AC_CONFIG_LINKS(tests/conftest.py:tests/conftest.py) +AC_CONFIG_LINKS(tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt:tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt) +AC_CONFIG_LINKS(tests/hmc/Test_hmc_IwasakiGauge_expected.txt:tests/hmc/Test_hmc_IwasakiGauge_expected.txt) +AC_CONFIG_LINKS(tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt:tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt) + AC_OUTPUT echo "" diff --git a/tests/hmc/Test_hmc_IwasakiGauge.cc b/tests/hmc/Test_hmc_IwasakiGauge.cc index 66345bc03e..eaf68b639f 100644 --- a/tests/hmc/Test_hmc_IwasakiGauge.cc +++ b/tests/hmc/Test_hmc_IwasakiGauge.cc @@ -51,7 +51,7 @@ int main(int argc, char **argv) { CheckpointerParameters CPparams; CPparams.config_prefix = "ckpoint_lat"; CPparams.rng_prefix = "ckpoint_rng"; - CPparams.saveInterval = 20; + CPparams.saveInterval = 1; CPparams.format = "IEEE64BIG"; TheHMC.Resources.LoadNerscCheckpointer(CPparams); diff --git a/tests/hmc/Test_hmc_IwasakiGauge_expected.txt b/tests/hmc/Test_hmc_IwasakiGauge_expected.txt index 9391cc8c3d..6726a0ed3b 100644 --- a/tests/hmc/Test_hmc_IwasakiGauge_expected.txt +++ b/tests/hmc/Test_hmc_IwasakiGauge_expected.txt @@ -1 +1,3 @@ -8.8.8.8 1.1.1.1 0.269298793 633bf471 3a22ad20 +1 8.8.8.8 1.1.1.1 4 20 1.0 GPU 0.269298793 633bf471 3a22ad20 -- Mashy's (probably local) +2 8.8.8.8 1.1.1.1 1 20 1.0 CPU 0.269298793 633bf471 3a116d60 +3 8.8.8.8 1.1.1.1 4 20 1.0 CPU 0.269298793 633bf471 3a116d60 diff --git a/tests/hmc/Test_hmc_WilsonFermionGauge.cc b/tests/hmc/Test_hmc_WilsonFermionGauge.cc index a0c43c515b..c89d2a87da 100644 --- a/tests/hmc/Test_hmc_WilsonFermionGauge.cc +++ b/tests/hmc/Test_hmc_WilsonFermionGauge.cc @@ -59,7 +59,7 @@ int main(int argc, char **argv) { CheckpointerParameters CPparams; CPparams.config_prefix = "ckpoint_lat"; CPparams.rng_prefix = "ckpoint_rng"; - CPparams.saveInterval = 5; + CPparams.saveInterval = 1; CPparams.format = "IEEE64BIG"; TheHMC.Resources.LoadNerscCheckpointer(CPparams); diff --git a/tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt b/tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt index 6b53130bfa..93f50e1d07 100644 --- a/tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt +++ b/tests/hmc/Test_hmc_WilsonFermionGauge_expected.txt @@ -1 +1,3 @@ -8.8.8.8 1.1.1.1 0.254950222 7f25d41 9d40279f +1 8.8.8.8 1.1.1.1 1 20 1.0 CPU 0.254950222 7f25d41 9d4e34e3 +2 8.8.8.8 1.1.1.1 4 20 1.0 CPU 0.254950222 7f25d41 9d63f203 +3 8.8.8.8 1.1.1.1 4 20 1.0 GPU 0.254950222 7f25d41 9d40279f -- Mashy's (probably local) diff --git a/tests/run_regression_test.py b/tests/run_regression_test.py index 935d85c45e..81bfa400d2 100644 --- a/tests/run_regression_test.py +++ b/tests/run_regression_test.py @@ -122,9 +122,9 @@ def test_outputs(test_name, expected_line, cleanup_files): print(f"Running {test_name} for test parameters: ", test_parameters) if test_parameters['nthreads'] == '0': - result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 > output.txt"], shell=True, encoding="text") - else: - result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 --threads {test_parameters['nthreads']} > output.txt"], shell=True, encoding="text") + print("Running with 1 thread: --threads 1") + test_parameters['nthreads'] = '1' + result = subprocess.run([f"./{test_name} --grid {test_parameters['grid']} --mpi {test_parameters['mpi']} --Thermalizations 0 --Trajectories 1 --threads {test_parameters['nthreads']} > output.txt"], shell=True, encoding="text") plaquette, checksum_rng, checksum_lat = read_output(test_parameters) if (checksum_rng is None) or (checksum_lat is None) or (plaquette is None): pytest.fail("Error reading values from output file. Make sure you compile the test with CPparams.saveInterval=1 in order to produce the required output.") diff --git a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt index 26b0d5ac9c..8d7e00a27e 100644 --- a/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt +++ b/tests/sp2n/Test_hmc_Sp_WilsonFundFermionGauge_expected.txt @@ -1,4 +1,4 @@ -1 8.8.8.8 1.1.1.1 0 36 1.0 CPU 0.281507042 922c392f 716ec776 +1 8.8.8.8 1.1.1.1 4 36 1.0 CPU 0.281507042 922c392f 7047fcf6 2 8.8.8.8 1.1.1.1 1 36 1.0 CPU 0.281507042 922c392f 6a8bb5f6 -3 8.8.8.8 1.1.1.1 0 2 0.1 CPU 0.0256253844 922c392f d1e4cc1c +3 8.8.8.8 1.1.1.1 4 2 0.1 CPU 0.0256253844 922c392f d1e371ab 4 8.8.8.8 1.1.1.1 1 2 0.1 CPU 0.0256253844 922c392f d1e4f305 \ No newline at end of file