From a2acf94b885bdc588f934db572ebcda8d5dd660d Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sun, 15 Mar 2026 19:55:23 -0500 Subject: [PATCH 1/6] test: initial commit of e2e tests --- libbenbot/src/engine/Printing.cpp | 5 +- libchess/src/game/Position.cpp | 2 + tests/CMakeLists.txt | 1 + tests/e2e/CMakeLists.txt | 19 ++ tests/e2e/README.md | 4 + tests/e2e/e2e.py | 483 ++++++++++++++++++++++++++++++ tests/e2e/testing.py | 363 ++++++++++++++++++++++ 7 files changed, 875 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/CMakeLists.txt create mode 100644 tests/e2e/README.md create mode 100644 tests/e2e/e2e.py create mode 100644 tests/e2e/testing.py diff --git a/libbenbot/src/engine/Printing.cpp b/libbenbot/src/engine/Printing.cpp index e3cf45b1..9890db36 100644 --- a/libbenbot/src/engine/Printing.cpp +++ b/libbenbot/src/engine/Printing.cpp @@ -151,14 +151,15 @@ void Engine::print_current_position(const string_view arguments) const println(""); - print_labeled_info("FEN: ", chess::notation::to_fen(pos)); + print_labeled_info("FEN: ", chess::notation::to_fen(pos)); + print_labeled_info("XFEN: ", chess::notation::to_fen(pos, false)); print_labeled_info("Zobrist key: ", std::format("{}", pos.hash)); - print_labeled_info("Static eval: ", std::format("{}", eval::evaluate(pos))); searcher.context.probe_transposition_table(pos) .transform([this](const TTData& data) { + println(""); print_labeled_info( "TT hit: ", std::format( diff --git a/libchess/src/game/Position.cpp b/libchess/src/game/Position.cpp index 2a38fce4..13902313 100644 --- a/libchess/src/game/Position.cpp +++ b/libchess/src/game/Position.cpp @@ -363,6 +363,8 @@ auto Position::is_illegal() const -> std::optional magic_enum::enum_name(sideToMove)); } + // TODO: our king cannot be in check by more than 2 pieces + return std::nullopt; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 76f7bfb1..32da7a30 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,6 +38,7 @@ endfunction () configure_file (CTestCustom.cmake "${BenBot_BINARY_DIR}/CTestCustom.cmake" @ONLY) +add_subdirectory (e2e) add_subdirectory (fastchess) add_subdirectory (perft) add_subdirectory (position-solver) diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt new file mode 100644 index 00000000..8bcb3445 --- /dev/null +++ b/tests/e2e/CMakeLists.txt @@ -0,0 +1,19 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +add_feature_info ( + benbot_e2e_testing "TARGET Python::Interpreter" "BenBot end-to-end & integration testing" +) + +if (NOT TARGET Python::Interpreter) + return () +endif () diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..22025786 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,4 @@ +# End-to-end testing + +This directory defines tests that are scripted to test the engine +executable in various scenarios. diff --git a/tests/e2e/e2e.py b/tests/e2e/e2e.py new file mode 100644 index 00000000..44ebae20 --- /dev/null +++ b/tests/e2e/e2e.py @@ -0,0 +1,483 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +import argparse +import re +import sys +import pathlib +import os +import fnmatch + +from testing import ( + EPD, + BenBot as Engine, + MiniTestFramework, + OrderedClassMembers, +) + +PATH = pathlib.Path(__file__).parent.resolve() +CWD = os.getcwd() + + +def get_threads(): + if args.valgrind_thread or args.sanitizer_thread: + return 2 + return 1 + + +def get_path(): + return os.path.abspath(os.path.join(CWD, args.stockfish_path)) + + +def postfix_check(output): + if args.sanitizer_undefined: + for idx, line in enumerate(output): + if "runtime error:" in line: + # print next possible 50 lines + for i in range(50): + debug_idx = idx + i + if debug_idx < len(output): + print(output[debug_idx]) + return False + + if args.sanitizer_thread: + for idx, line in enumerate(output): + if "WARNING: ThreadSanitizer:" in line: + # print next possible 50 lines + for i in range(50): + debug_idx = idx + i + if debug_idx < len(output): + print(output[debug_idx]) + return False + + return True + + +def BenBot(*args, **kwargs): + return Engine(get_path(), *args, **kwargs) + + +class TestCLI(metaclass=OrderedClassMembers): + def beforeAll(self): + pass + + def after_all(self): + pass + + def beforeEach(self): + self.stockfish = None + + def afterEach(self): + assert postfix_check(self.stockfish.get_output()) == True + self.stockfish.clear_output() + + def test_go_nodes_1000(self): + self.stockfish = BenBot("go nodes 1000".split(" "), True) + assert self.stockfish.process.returncode == 0 + + def test_go_depth_10(self): + self.stockfish = BenBot("go depth 10".split(" "), True) + assert self.stockfish.process.returncode == 0 + + def test_go_perft_4(self): + self.stockfish = BenBot("perft 4 json".split(" "), True) + assert self.stockfish.process.returncode == 0 + + def test_go_movetime_1000(self): + self.stockfish = BenBot("go movetime 1000".split(" "), True) + assert self.stockfish.process.returncode == 0 + + def test_go_wtime_8000_btime_8000_winc_500_binc_500(self): + self.stockfish = BenBot( + "go wtime 8000 btime 8000 winc 500 binc 500".split(" "), + True, + ) + assert self.stockfish.process.returncode == 0 + + def test_go_wtime_1000_btime_1000_winc_0_binc_0(self): + self.stockfish = BenBot( + "go wtime 1000 btime 1000 winc 0 binc 0".split(" "), + True, + ) + assert self.stockfish.process.returncode == 0 + + def test_go_wtime_1000_btime_1000_winc_0_binc_0_movestogo_5(self): + self.stockfish = BenBot( + "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5".split(" "), + True, + ) + assert self.stockfish.process.returncode == 0 + + def test_go_movetime_200(self): + self.stockfish = BenBot("go movetime 200".split(" "), True) + assert self.stockfish.process.returncode == 0 + + def test_go_nodes_20000_searchmoves_e2e4_d2d4(self): + self.stockfish = BenBot("go nodes 20000 searchmoves e2e4 d2d4".split(" "), True) + assert self.stockfish.process.returncode == 0 + + def test_bench(self): + self.stockfish = BenBot( + f"bench".split(" "), + True, + ) + assert self.stockfish.process.returncode == 0 + + def test_bench_2(self): + self.stockfish = BenBot( + f"bench 3 {os.path.join(PATH, 'bench_tmp.epd')} depth".split(" "), + True, + ) + assert self.stockfish.process.returncode == 0 + + def test_showpos(self): + self.stockfish = BenBot("showpos".split(" "), True) + assert self.stockfish.process.returncode == 0 + + def test_compiler(self): + self.stockfish = BenBot("compiler".split(" "), True) + assert self.stockfish.process.returncode == 0 + + def test_uci(self): + self.stockfish = BenBot("uci".split(" "), True) + assert self.stockfish.process.returncode == 0 + + +class TestInteractive(metaclass=OrderedClassMembers): + def beforeAll(self): + self.stockfish = BenBot() + + def after_all(self): + self.stockfish.quit() + assert self.stockfish.close() == 0 + + def afterEach(self): + assert postfix_check(self.stockfish.get_output()) == True + self.stockfish.clear_output() + + def test_uci_command(self): + self.stockfish.send_command("uci") + self.stockfish.equals("uciok") + + def test_set_threads_option(self): + self.stockfish.send_command(f"setoption name Threads value {get_threads()}") + + def test_ucinewgame_and_startpos_nodes_1000(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command("position startpos") + self.stockfish.send_command("go nodes 1000") + self.stockfish.starts_with("bestmove") + + def test_ucinewgame_and_startpos_moves(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command("position startpos moves e2e4 e7e6") + self.stockfish.send_command("go nodes 1000") + self.stockfish.starts_with("bestmove") + + def test_fen_position_1(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") + self.stockfish.send_command("go nodes 1000") + self.stockfish.starts_with("bestmove") + + def test_fen_position_2_flip(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") + self.stockfish.send_command("flip") + self.stockfish.send_command("go nodes 1000") + self.stockfish.starts_with("bestmove") + + def test_depth_5_with_callback(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command("position startpos") + self.stockfish.send_command("go depth 5") + + def callback(output): + regex = r"info depth \d+ seldepth \d+ score cp -?\d+ time \d+ hashfull \d+ nodes \d+ nps \d+ tbhits \d+ pv" + if output.startswith("info depth") and not re.match(regex, output): + assert False + if output.startswith("bestmove"): + return True + return False + + self.stockfish.check_output(callback) + + def test_ucinewgame_and_go_depth_4(self): + total_depth = 4 + + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command("position startpos") + self.stockfish.send_command(f"go depth {total_depth}") + + depth = 0 + + def callback(output): + nonlocal depth + nonlocal total_depth + + if output.startswith("info depth"): + depth += 1 + + regex = rf"info depth {depth} seldepth \d+ score cp -?\d+ time \d+ hashfull \d+ nodes \d+ nps \d+ tbhits \d+ pv" + + if not re.match(regex, output): + assert False + + if output.startswith("bestmove"): + assert depth == total_depth + return True + + return False + + self.stockfish.check_output(callback) + + def test_clear_hash(self): + self.stockfish.send_command("setoption name Clear Hash") + + def test_fen_position_mate_1(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6" + ) + self.stockfish.send_command("go depth 10") + + self.stockfish.expect("* score mate 1 * pv d5e6") + self.stockfish.equals("bestmove d5e6") + + def test_fen_position_mate_minus_1(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -" + ) + self.stockfish.send_command("go depth 10") + self.stockfish.expect("* score mate -1 *") + self.stockfish.starts_with("bestmove") + + def test_fen_position_fixed_node(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 5K2/8/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1" + ) + self.stockfish.send_command("go nodes 10000") + self.stockfish.starts_with("bestmove") + + def test_fen_position_with_mate_go_depth(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -" + ) + self.stockfish.send_command("go depth 18 searchmoves c6d7") + self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5") + + self.stockfish.starts_with("bestmove") + + def test_fen_position_with_mate_go_mate(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -" + ) + self.stockfish.send_command("go mate 2 searchmoves c6d7") + self.stockfish.expect("* score mate 2 * pv c6d7 *") + + self.stockfish.starts_with("bestmove") + + def test_fen_position_with_mate_go_nodes(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -" + ) + self.stockfish.send_command("go nodes 500000 searchmoves c6d7") + self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5") + + self.stockfish.starts_with("bestmove") + + def test_fen_position_depth_8(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen r1b2r1k/pp1p2pp/2p5/2B1q3/8/8/P1PN2PP/R4RK1 w - - 0 18" + ) + self.stockfish.send_command("go depth 8") + self.stockfish.contains("score mate 1") + + self.stockfish.starts_with("bestmove") + + def test_fen_position_with_mate_go_depth_and_promotion(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q" + ) + self.stockfish.send_command("go depth 13") + self.stockfish.expect("* score mate 1 * pv f7f5") + self.stockfish.starts_with("bestmove f7f5") + + def test_fen_position_with_mate_go_depth_and_searchmoves(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -" + ) + self.stockfish.send_command("go depth 18 searchmoves c6d7") + self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5") + + self.stockfish.starts_with("bestmove c6d7") + + def test_fen_position_with_moves_with_mate_go_depth_and_searchmoves(self): + self.stockfish.send_command("ucinewgame") + self.stockfish.send_command( + "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7" + ) + self.stockfish.send_command("go depth 18 searchmoves e3e2") + self.stockfish.expect("* score mate -1 * pv e3e2 f7f5") + self.stockfish.starts_with("bestmove e3e2") + + +class TestEnPassantSanitization(metaclass=OrderedClassMembers): + def beforeAll(self): + self.stockfish = BenBot() + + def after_all(self): + self.stockfish.quit() + assert self.stockfish.close() == 0 + + def afterEach(self): + assert postfix_check(self.stockfish.get_output()) == True + self.stockfish.clear_output() + + def test_position_1(self): + self.stockfish.send_command( + "position fen rnbqkbnr/ppp1p1pp/5p2/3pP3/8/8/PPPP1PPP/RNBQKBNR w kq d6 0 3" + ) + self.stockfish.send_command("showpos") + + self.stockfish.expect_for_line_matching( + "FEN*", "*rnbqkbnr/ppp1p1pp/5p2/3pP3/8/8/PPPP1PPP/RNBQKBNR w kq d6 0 3*" + ) + + def test_position_2(self): + self.stockfish.send_command("position fen k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1") + self.stockfish.send_command("showpos") + + self.stockfish.expect_for_line_matching( + "FEN*", "*k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1*" + ) + + # def test_position_3(self): + # self.stockfish.send_command("position fen k1r5/8/8/1pP5/2K5/8/8/8 w - b6 0 1") + # self.stockfish.send_command("showpos") + # + # self.stockfish.expect_for_line_matching( + # "XFEN*", "*k1r5/8/8/1pP5/2K5/8/8/8 w - - 0 1*" + # ) + + # def test_position_4(self): + # self.stockfish.send_command("position fen k1r5/8/8/1pP5/8/2K5/8/8 w - b6 0 1") + # self.stockfish.send_command("showpos") + # + # self.stockfish.expect_for_line_matching( + # "XFEN*", "*k1r5/8/8/1pP5/8/2K5/8/8 w - - 0 1*" + # ) + + def test_position_5(self): + self.stockfish.send_command("position fen k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1") + self.stockfish.send_command("showpos") + + self.stockfish.expect_for_line_matching( + "FEN*", "*k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1*" + ) + + def test_position_6(self): + self.stockfish.send_command("position fen k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1") + self.stockfish.send_command("showpos") + + self.stockfish.expect_for_line_matching( + "FEN*", "*k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1*" + ) + + def test_position_7(self): + self.stockfish.send_command("position fen k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1") + self.stockfish.send_command("showpos") + + self.stockfish.expect_for_line_matching( + "FEN*", "*k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1*" + ) + + # def test_position_8(self): + # self.stockfish.send_command("position fen k7/b5b1/8/2PpP3/3K4/8/8/8 w - d6 0 1") + # self.stockfish.send_command("showpos") + # + # self.stockfish.expect_for_line_matching( + # "XFEN*", "*k7/b5b1/8/2PpP3/3K4/8/8/8 w - - 0 1*" + # ) + + # def test_position_9(self): + # self.stockfish.send_command("position fen k7/8/8/r2pPK2/8/8/8/8 w - d6 0 1") + # self.stockfish.send_command("showpos") + # + # self.stockfish.expect_for_line_matching( + # "XFEN*", "*k7/8/8/r2pPK2/8/8/8/8 w - - 0 1*" + # ) + + def test_position_10(self): + self.stockfish.send_command("position fen k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1") + self.stockfish.send_command("showpos") + + self.stockfish.expect_for_line_matching( + "FEN*", "*k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1*" + ) + + def test_position_11(self): + self.stockfish.send_command("position fen kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1") + self.stockfish.send_command("showpos") + + self.stockfish.expect_for_line_matching( + "FEN*", "*kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1*" + ) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run Stockfish with testing options") + parser.add_argument("--valgrind", action="store_true", help="Run valgrind testing") + parser.add_argument( + "--valgrind-thread", action="store_true", help="Run valgrind-thread testing" + ) + parser.add_argument( + "--sanitizer-undefined", + action="store_true", + help="Run sanitizer-undefined testing", + ) + parser.add_argument( + "--sanitizer-thread", action="store_true", help="Run sanitizer-thread testing" + ) + + parser.add_argument( + "--none", action="store_true", help="Run without any testing options" + ) + parser.add_argument("stockfish_path", type=str, help="Path to Stockfish binary") + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + EPD.create_bench_epd() + + framework = MiniTestFramework() + + # Each test suite will be run inside a temporary directory + framework.run([TestCLI, TestInteractive, TestEnPassantSanitization]) + + EPD.delete_bench_epd() + + if framework.has_failed(): + sys.exit(1) + + sys.exit(0) diff --git a/tests/e2e/testing.py b/tests/e2e/testing.py new file mode 100644 index 00000000..7fbf5c74 --- /dev/null +++ b/tests/e2e/testing.py @@ -0,0 +1,363 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +import subprocess +from typing import List +import os +import collections +import time +import sys +import traceback +import fnmatch +from functools import wraps +from contextlib import redirect_stdout +import io +import pathlib +import concurrent.futures +import tempfile + +CYAN_COLOR = "\033[36m" +GRAY_COLOR = "\033[2m" +RED_COLOR = "\033[31m" +GREEN_COLOR = "\033[32m" +RESET_COLOR = "\033[0m" +WHITE_BOLD = "\033[1m" + +MAX_TIMEOUT = 60 * 5 + +PATH = pathlib.Path(__file__).parent.resolve() + + +class EPD: + @staticmethod + def create_bench_epd(): + with open(f"{os.path.join(PATH,'bench_tmp.epd')}", "w") as f: + f.write(""" +Rn6/1rbq1bk1/2p2n1p/2Bp1p2/3Pp1pP/1N2P1P1/2Q1NPB1/6K1 w - - 2 26 +rnbqkb1r/ppp1pp2/5n1p/3p2p1/P2PP3/5P2/1PP3PP/RNBQKBNR w KQkq - 0 3 +3qnrk1/4bp1p/1p2p1pP/p2bN3/1P1P1B2/P2BQ3/5PP1/4R1K1 w - - 9 28 +r4rk1/1b2ppbp/pq4pn/2pp1PB1/1p2P3/1P1P1NN1/1PP3PP/R2Q1RK1 w - - 0 13 +""") + + @staticmethod + def delete_bench_epd(): + os.remove(f"{os.path.join(PATH,'bench_tmp.epd')}") + + +class OrderedClassMembers(type): + @classmethod + def __prepare__(self, name, bases): + return collections.OrderedDict() + + def __new__(self, name, bases, classdict): + classdict["__ordered__"] = [ + key for key in classdict.keys() if key not in ("__module__", "__qualname__") + ] + return type.__new__(self, name, bases, classdict) + + +class TimeoutException(Exception): + def __init__(self, message: str, timeout: int): + self.message = message + self.timeout = timeout + + +class UnexpectedOutputException(Exception): + def __init__(self, actual: str, expected: str): + self.actual = actual + self.expected = expected + + +def timeout_decorator(timeout: float): + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit(func, *args, **kwargs) + try: + result = future.result(timeout=timeout) + except concurrent.futures.TimeoutError: + raise TimeoutException( + f"Function {func.__name__} timed out after {timeout} seconds", + int(timeout), + ) + return result + + return wrapper + + return decorator + + +class MiniTestFramework: + def __init__(self): + self.passed_test_suites = 0 + self.failed_test_suites = 0 + self.passed_tests = 0 + self.failed_tests = 0 + self.start_time = None + self.stop_on_failure = True + + def has_failed(self) -> bool: + return self.failed_test_suites > 0 + + def run(self, classes: List[type]) -> bool: + self.start_time = time.time() + + for test_class in classes: + with tempfile.TemporaryDirectory() as tmpdirname: + original_cwd = os.getcwd() + os.chdir(tmpdirname) + + try: + if self.__run(test_class): + self.failed_test_suites += 1 + else: + self.passed_test_suites += 1 + except Exception as e: + self.failed_test_suites += 1 + print(f"\n{RED_COLOR}Error: {e}{RESET_COLOR}") + finally: + os.chdir(original_cwd) + + self.__print_summary(round(time.time() - self.start_time, 2)) + return self.has_failed() + + def __run(self, test_class) -> bool: + test_instance = test_class() + test_name = test_instance.__class__.__name__ + test_methods = [m for m in test_instance.__ordered__ if m.startswith("test_")] + + print(f"\nTest Suite: {test_name}") + + if hasattr(test_instance, "beforeAll"): + test_instance.beforeAll() + + fails = 0 + + for method in test_methods: + fails += self.__run_test_method(test_instance, method) + + if hasattr(test_instance, "after_all"): + test_instance.after_all() + + self.failed_tests += fails + + return fails > 0 + + def __run_test_method(self, test_instance, method: str) -> int: + print(f" Running {method}... \r", end="", flush=True) + + buffer = io.StringIO() + fails = 0 + + try: + t0 = time.time() + + with redirect_stdout(buffer): + if hasattr(test_instance, "beforeEach"): + test_instance.beforeEach() + + getattr(test_instance, method)() + + if hasattr(test_instance, "afterEach"): + test_instance.afterEach() + + duration = time.time() - t0 + + self.print_success(f" {method} ({duration * 1000:.2f}ms)") + self.passed_tests += 1 + except Exception as e: + if isinstance(e, TimeoutException): + self.print_failure( + f" {method} (hit execution limit of {e.timeout} seconds)" + ) + + if isinstance(e, UnexpectedOutputException): + self.print_failure( + f' {method} encountered unexpected output: "{e.actual}" when output matching "{e.expected}" was expected' + ) + + if isinstance(e, AssertionError): + self.__handle_assertion_error(t0, method) + + if self.stop_on_failure: + self.__print_buffer_output(buffer) + raise e + + fails += 1 + finally: + self.__print_buffer_output(buffer) + + return fails + + def __handle_assertion_error(self, start_time, method: str): + duration = time.time() - start_time + self.print_failure(f" {method} ({duration * 1000:.2f}ms)") + traceback_output = "".join(traceback.format_tb(sys.exc_info()[2])) + + colored_traceback = "\n".join( + f" {CYAN_COLOR}{line}{RESET_COLOR}" + for line in traceback_output.splitlines() + ) + + print(colored_traceback) + + def __print_buffer_output(self, buffer: io.StringIO): + output = buffer.getvalue() + if output: + indented_output = "\n".join(f" {line}" for line in output.splitlines()) + print(f" {RED_COLOR}⎯⎯⎯⎯⎯OUTPUT⎯⎯⎯⎯⎯{RESET_COLOR}") + print(f"{GRAY_COLOR}{indented_output}{RESET_COLOR}") + print(f" {RED_COLOR}⎯⎯⎯⎯⎯OUTPUT⎯⎯⎯⎯⎯{RESET_COLOR}") + + def __print_summary(self, duration: float): + print(f"\n{WHITE_BOLD}Test Summary{RESET_COLOR}\n") + print( + f" Test Suites: {GREEN_COLOR}{self.passed_test_suites} passed{RESET_COLOR}, {RED_COLOR}{self.failed_test_suites} failed{RESET_COLOR}, {self.passed_test_suites + self.failed_test_suites} total" + ) + print( + f" Tests: {GREEN_COLOR}{self.passed_tests} passed{RESET_COLOR}, {RED_COLOR}{self.failed_tests} failed{RESET_COLOR}, {self.passed_tests + self.failed_tests} total" + ) + print(f" Time: {duration}s\n") + + def print_failure(self, add: str): + print(f" {RED_COLOR}✗{RESET_COLOR}{add}", flush=True) + + def print_success(self, add: str): + print(f" {GREEN_COLOR}✓{RESET_COLOR}{add}", flush=True) + + +class BenBot: + def __init__( + self, + path: str, + args: List[str] = [], + cli: bool = False, + ): + self.path = path + self.process = None + self.args = args + self.cli = cli + self.output = [] + + self.start() + + def _check_process_alive(self): + if not self.process or self.process.poll() is not None: + print("\n".join(self.output)) + raise RuntimeError("Stockfish process has terminated") + + def start(self): + if self.cli: + self.process = subprocess.run( + [self.path] + self.args + ["--no-loop"], + capture_output=True, + text=True, + ) + + if self.process.returncode != 0: + print(self.process.stdout) + print(self.process.stderr) + print(f"Process failed with return code {self.process.returncode}") + + return + + self.process = subprocess.Popen( + [self.path] + self.args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + bufsize=1, + ) + + def setoption(self, name: str, value: str): + self.send_command(f"setoption name {name} value {value}") + + def send_command(self, command: str): + if not self.process: + raise RuntimeError("BenBot process is not started") + + self._check_process_alive() + + self.process.stdin.write(command + "\n") + self.process.stdin.flush() + + @timeout_decorator(MAX_TIMEOUT) + def equals(self, expected_output: str): + for line in self.readline(): + if line == expected_output: + return + + @timeout_decorator(MAX_TIMEOUT) + def expect(self, expected_output: str): + for line in self.readline(): + if fnmatch.fnmatch(line, expected_output): + return + + @timeout_decorator(MAX_TIMEOUT) + def contains(self, expected_output: str): + for line in self.readline(): + if expected_output in line: + return + + @timeout_decorator(MAX_TIMEOUT) + def starts_with(self, expected_output: str): + for line in self.readline(): + if line.startswith(expected_output): + return + + @timeout_decorator(MAX_TIMEOUT) + def check_output(self, callback): + if not callback: + raise ValueError("Callback function is required") + + for line in self.readline(): + if callback(line) == True: + return + + @timeout_decorator(MAX_TIMEOUT) + def expect_for_line_matching(self, line_match: str, expected: str): + for line in self.readline(): + if fnmatch.fnmatch(line, line_match): + if fnmatch.fnmatch(line, expected): + break + else: + raise UnexpectedOutputException(line, expected) + + def readline(self): + if not self.process: + raise RuntimeError("BenBot process is not started") + + while True: + self._check_process_alive() + line = self.process.stdout.readline().strip() + self.output.append(line) + + yield line + + def clear_output(self): + self.output = [] + + def get_output(self) -> List[str]: + return self.output + + def quit(self): + self.send_command("quit") + + def close(self): + if self.process: + self.process.stdin.close() + self.process.stdout.close() + return self.process.wait() + + return 0 From 9ad88cd7f2cba335230c168d7ce8858d06bbade7 Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sun, 15 Mar 2026 20:26:37 -0500 Subject: [PATCH 2/6] test: updating e2e tests --- tests/e2e/e2e.py | 141 +++++++++---------------------------------- tests/e2e/testing.py | 48 ++++++--------- 2 files changed, 47 insertions(+), 142 deletions(-) diff --git a/tests/e2e/e2e.py b/tests/e2e/e2e.py index 44ebae20..2feeb369 100644 --- a/tests/e2e/e2e.py +++ b/tests/e2e/e2e.py @@ -15,10 +15,8 @@ import sys import pathlib import os -import fnmatch from testing import ( - EPD, BenBot as Engine, MiniTestFramework, OrderedClassMembers, @@ -28,140 +26,85 @@ CWD = os.getcwd() -def get_threads(): - if args.valgrind_thread or args.sanitizer_thread: - return 2 - return 1 - - def get_path(): return os.path.abspath(os.path.join(CWD, args.stockfish_path)) -def postfix_check(output): - if args.sanitizer_undefined: - for idx, line in enumerate(output): - if "runtime error:" in line: - # print next possible 50 lines - for i in range(50): - debug_idx = idx + i - if debug_idx < len(output): - print(output[debug_idx]) - return False - - if args.sanitizer_thread: - for idx, line in enumerate(output): - if "WARNING: ThreadSanitizer:" in line: - # print next possible 50 lines - for i in range(50): - debug_idx = idx + i - if debug_idx < len(output): - print(output[debug_idx]) - return False - - return True - - def BenBot(*args, **kwargs): return Engine(get_path(), *args, **kwargs) class TestCLI(metaclass=OrderedClassMembers): - def beforeAll(self): - pass - - def after_all(self): - pass - - def beforeEach(self): - self.stockfish = None - - def afterEach(self): - assert postfix_check(self.stockfish.get_output()) == True - self.stockfish.clear_output() - def test_go_nodes_1000(self): - self.stockfish = BenBot("go nodes 1000".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("go nodes 1000".split(" "), True) + assert engine.process.returncode == 0 def test_go_depth_10(self): - self.stockfish = BenBot("go depth 10".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("go depth 10".split(" "), True) + assert engine.process.returncode == 0 def test_go_perft_4(self): - self.stockfish = BenBot("perft 4 json".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("perft 4 json".split(" "), True) + assert engine.process.returncode == 0 def test_go_movetime_1000(self): - self.stockfish = BenBot("go movetime 1000".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("go movetime 1000".split(" "), True) + assert engine.process.returncode == 0 def test_go_wtime_8000_btime_8000_winc_500_binc_500(self): - self.stockfish = BenBot( + engine = BenBot( "go wtime 8000 btime 8000 winc 500 binc 500".split(" "), True, ) - assert self.stockfish.process.returncode == 0 + assert engine.process.returncode == 0 def test_go_wtime_1000_btime_1000_winc_0_binc_0(self): - self.stockfish = BenBot( + engine = BenBot( "go wtime 1000 btime 1000 winc 0 binc 0".split(" "), True, ) - assert self.stockfish.process.returncode == 0 + assert engine.process.returncode == 0 def test_go_wtime_1000_btime_1000_winc_0_binc_0_movestogo_5(self): - self.stockfish = BenBot( + engine = BenBot( "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5".split(" "), True, ) - assert self.stockfish.process.returncode == 0 + assert engine.process.returncode == 0 def test_go_movetime_200(self): - self.stockfish = BenBot("go movetime 200".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("go movetime 200".split(" "), True) + assert engine.process.returncode == 0 def test_go_nodes_20000_searchmoves_e2e4_d2d4(self): - self.stockfish = BenBot("go nodes 20000 searchmoves e2e4 d2d4".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("go nodes 20000 searchmoves e2e4 d2d4".split(" "), True) + assert engine.process.returncode == 0 def test_bench(self): - self.stockfish = BenBot( + engine = BenBot( f"bench".split(" "), True, ) - assert self.stockfish.process.returncode == 0 - - def test_bench_2(self): - self.stockfish = BenBot( - f"bench 3 {os.path.join(PATH, 'bench_tmp.epd')} depth".split(" "), - True, - ) - assert self.stockfish.process.returncode == 0 + assert engine.process.returncode == 0 def test_showpos(self): - self.stockfish = BenBot("showpos".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("showpos".split(" "), True) + assert engine.process.returncode == 0 def test_compiler(self): - self.stockfish = BenBot("compiler".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("compiler".split(" "), True) + assert engine.process.returncode == 0 def test_uci(self): - self.stockfish = BenBot("uci".split(" "), True) - assert self.stockfish.process.returncode == 0 + engine = BenBot("uci".split(" "), True) + assert engine.process.returncode == 0 class TestInteractive(metaclass=OrderedClassMembers): - def beforeAll(self): + def __init__(self): self.stockfish = BenBot() - def after_all(self): - self.stockfish.quit() - assert self.stockfish.close() == 0 - def afterEach(self): - assert postfix_check(self.stockfish.get_output()) == True self.stockfish.clear_output() def test_uci_command(self): @@ -169,7 +112,7 @@ def test_uci_command(self): self.stockfish.equals("uciok") def test_set_threads_option(self): - self.stockfish.send_command(f"setoption name Threads value {get_threads()}") + self.stockfish.send_command(f"setoption name Threads value 1") def test_ucinewgame_and_startpos_nodes_1000(self): self.stockfish.send_command("ucinewgame") @@ -340,15 +283,10 @@ def test_fen_position_with_moves_with_mate_go_depth_and_searchmoves(self): class TestEnPassantSanitization(metaclass=OrderedClassMembers): - def beforeAll(self): + def __init__(self): self.stockfish = BenBot() - def after_all(self): - self.stockfish.quit() - assert self.stockfish.close() == 0 - def afterEach(self): - assert postfix_check(self.stockfish.get_output()) == True self.stockfish.clear_output() def test_position_1(self): @@ -444,22 +382,7 @@ def test_position_11(self): def parse_args(): parser = argparse.ArgumentParser(description="Run Stockfish with testing options") - parser.add_argument("--valgrind", action="store_true", help="Run valgrind testing") - parser.add_argument( - "--valgrind-thread", action="store_true", help="Run valgrind-thread testing" - ) - parser.add_argument( - "--sanitizer-undefined", - action="store_true", - help="Run sanitizer-undefined testing", - ) - parser.add_argument( - "--sanitizer-thread", action="store_true", help="Run sanitizer-thread testing" - ) - - parser.add_argument( - "--none", action="store_true", help="Run without any testing options" - ) + parser.add_argument("stockfish_path", type=str, help="Path to Stockfish binary") return parser.parse_args() @@ -468,15 +391,11 @@ def parse_args(): if __name__ == "__main__": args = parse_args() - EPD.create_bench_epd() - framework = MiniTestFramework() # Each test suite will be run inside a temporary directory framework.run([TestCLI, TestInteractive, TestEnPassantSanitization]) - EPD.delete_bench_epd() - if framework.has_failed(): sys.exit(1) diff --git a/tests/e2e/testing.py b/tests/e2e/testing.py index 7fbf5c74..1cc68126 100644 --- a/tests/e2e/testing.py +++ b/tests/e2e/testing.py @@ -11,6 +11,7 @@ # ====================================================================================== import subprocess +from subprocess import CompletedProcess from typing import List import os import collections @@ -21,7 +22,6 @@ from functools import wraps from contextlib import redirect_stdout import io -import pathlib import concurrent.futures import tempfile @@ -34,24 +34,6 @@ MAX_TIMEOUT = 60 * 5 -PATH = pathlib.Path(__file__).parent.resolve() - - -class EPD: - @staticmethod - def create_bench_epd(): - with open(f"{os.path.join(PATH,'bench_tmp.epd')}", "w") as f: - f.write(""" -Rn6/1rbq1bk1/2p2n1p/2Bp1p2/3Pp1pP/1N2P1P1/2Q1NPB1/6K1 w - - 2 26 -rnbqkb1r/ppp1pp2/5n1p/3p2p1/P2PP3/5P2/1PP3PP/RNBQKBNR w KQkq - 0 3 -3qnrk1/4bp1p/1p2p1pP/p2bN3/1P1P1B2/P2BQ3/5PP1/4R1K1 w - - 9 28 -r4rk1/1b2ppbp/pq4pn/2pp1PB1/1p2P3/1P1P1NN1/1PP3PP/R2Q1RK1 w - - 0 13 -""") - - @staticmethod - def delete_bench_epd(): - os.remove(f"{os.path.join(PATH,'bench_tmp.epd')}") - class OrderedClassMembers(type): @classmethod @@ -138,17 +120,11 @@ def __run(self, test_class) -> bool: print(f"\nTest Suite: {test_name}") - if hasattr(test_instance, "beforeAll"): - test_instance.beforeAll() - fails = 0 for method in test_methods: fails += self.__run_test_method(test_instance, method) - if hasattr(test_instance, "after_all"): - test_instance.after_all() - self.failed_tests += fails return fails > 0 @@ -211,7 +187,8 @@ def __handle_assertion_error(self, start_time, method: str): print(colored_traceback) - def __print_buffer_output(self, buffer: io.StringIO): + @staticmethod + def __print_buffer_output(buffer: io.StringIO): output = buffer.getvalue() if output: indented_output = "\n".join(f" {line}" for line in output.splitlines()) @@ -229,10 +206,12 @@ def __print_summary(self, duration: float): ) print(f" Time: {duration}s\n") - def print_failure(self, add: str): + @staticmethod + def print_failure(add: str): print(f" {RED_COLOR}✗{RESET_COLOR}{add}", flush=True) - def print_success(self, add: str): + @staticmethod + def print_success(add: str): print(f" {GREEN_COLOR}✓{RESET_COLOR}{add}", flush=True) @@ -251,6 +230,10 @@ def __init__( self.start() + def __del__(self): + self.quit() + assert self.close() == 0 + def _check_process_alive(self): if not self.process or self.process.poll() is not None: print("\n".join(self.output)) @@ -269,6 +252,8 @@ def start(self): print(self.process.stderr) print(f"Process failed with return code {self.process.returncode}") + self.process = None + return self.process = subprocess.Popen( @@ -285,7 +270,7 @@ def setoption(self, name: str, value: str): def send_command(self, command: str): if not self.process: - raise RuntimeError("BenBot process is not started") + raise RuntimeError("BenBot process is completed or not started") self._check_process_alive() @@ -336,7 +321,7 @@ def expect_for_line_matching(self, line_match: str, expected: str): def readline(self): if not self.process: - raise RuntimeError("BenBot process is not started") + raise RuntimeError("BenBot process is completed or not started") while True: self._check_process_alive() @@ -352,7 +337,8 @@ def get_output(self) -> List[str]: return self.output def quit(self): - self.send_command("quit") + if self.process: + self.send_command("quit") def close(self): if self.process: From faff5685dedac98f9f8bacee5205285e3252401a Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sun, 15 Mar 2026 21:03:06 -0500 Subject: [PATCH 3/6] fix: if EngineBase receives a go command before ucinewgame, subclass still gets a newgame() call before the go() call --- libchess/include/libchess/uci/EngineBase.hpp | 3 ++- libchess/src/uci/EngineBase.cpp | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/libchess/include/libchess/uci/EngineBase.hpp b/libchess/include/libchess/uci/EngineBase.hpp index 39b1f4a2..a17a576a 100644 --- a/libchess/include/libchess/uci/EngineBase.hpp +++ b/libchess/include/libchess/uci/EngineBase.hpp @@ -263,6 +263,7 @@ struct EngineBase { void handle_quit(); void handle_setpos(string_view arguments); void handle_setoption(string_view arguments); + void handle_go(string_view arguments); static_assert( std::atomic_bool::is_always_lock_free, @@ -313,7 +314,7 @@ struct EngineBase { .argsHelp = "[startpos|fen ] [moves ]" }, EngineCommand { .name = "go", - .action = [this](const string_view args) { go(parse_go_options(args, position)); }, + .action = [this](const string_view args) { handle_go(args); }, .description = "Start a search", .argsHelp = { } }, EngineCommand { diff --git a/libchess/src/uci/EngineBase.cpp b/libchess/src/uci/EngineBase.cpp index fd02678e..9d3ec6e1 100644 --- a/libchess/src/uci/EngineBase.cpp +++ b/libchess/src/uci/EngineBase.cpp @@ -140,6 +140,18 @@ void EngineBase::respond_to_newgame() new_game(not wasInitialized); } +void EngineBase::handle_go(const string_view arguments) +{ + // if we haven't received a ucinewgame yet, make sure the subclass + // gets its expected new_game() call before its first go() call + if (not initialized.exchange(true, memory_order_relaxed)) { + new_game(true); + } + + go( + parse_go_options(arguments, position)); +} + void EngineBase::handle_quit() { abort_search(); From 4156c49ce9194e020903dd0acf2876d3b84550ed Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sun, 15 Mar 2026 21:09:24 -0500 Subject: [PATCH 4/6] test: one-shot-cli tests --- tests/e2e/CMakeLists.txt | 7 +++++ tests/e2e/OneShotCLIs.py | 67 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 tests/e2e/OneShotCLIs.py diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index 8bcb3445..d3ee9ade 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -17,3 +17,10 @@ add_feature_info ( if (NOT TARGET Python::Interpreter) return () endif () + +add_test (NAME ben_bot.one_shot_clis + COMMAND Python::Interpreter "${CMAKE_CURRENT_LIST_DIR}/OneShotCLIs.py" + "$" +) + +set_tests_properties (ben_bot.one_shot_clis PROPERTIES REQUIRED_FILES "$") diff --git a/tests/e2e/OneShotCLIs.py b/tests/e2e/OneShotCLIs.py new file mode 100644 index 00000000..fb149ea1 --- /dev/null +++ b/tests/e2e/OneShotCLIs.py @@ -0,0 +1,67 @@ +# ====================================================================================== +# +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░ +# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓███████▓▒░ ░▒▓██████▓▒░ ░▒▓█▓▒░ +# +# ====================================================================================== + +import subprocess +import sys +from pathlib import Path + + +def run_oneshot_cli_test_suite(commands: list[str], benbot_path: Path): + num_passed = 0 + num_failed = 0 + + def run_benbot_command(command: str): + nonlocal num_passed + nonlocal num_failed + + print(f"Running command: '{command}'") + + process = subprocess.run( + [benbot_path] + command.split(" ") + ["--no-loop"], + capture_output=True, + text=True, + ) + + if process.returncode == 0: + num_passed += 1 + return + + print(process.stdout) + print(process.stderr) + print(f"Process failed with return code {process.returncode}") + num_failed += 1 + + for cmd in commands: + run_benbot_command(cmd) + + print(f"{num_passed} tests passed") + print(f"{num_failed} tests failed") + + exit(num_failed) + + +TEST_COMMANDS = [ + "go nodes 1000", + "go depth 10", + "perft 4 json", + "go movetime 1000", + "go wtime 8000 btime 8000 winc 500 binc 500", + "go wtime 1000 btime 1000 winc 0 binc 0", + "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5", + "go movetime 200", + "go nodes 20000 searchmoves e2e4 d2d4", + "showpos", + "compiler", + "uci", +] + +run_oneshot_cli_test_suite(TEST_COMMANDS, Path(sys.argv[1])) From 4c93df2bbe9d59ad6d25e56ae1cbeb55b35f5d03 Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sun, 15 Mar 2026 21:36:21 -0500 Subject: [PATCH 5/6] refactor: e2e test script --- tests/e2e/CMakeLists.txt | 8 +- tests/e2e/e2e.py | 314 +++++++++++++++------------------------ tests/e2e/testing.py | 5 +- 3 files changed, 128 insertions(+), 199 deletions(-) diff --git a/tests/e2e/CMakeLists.txt b/tests/e2e/CMakeLists.txt index d3ee9ade..032373f3 100644 --- a/tests/e2e/CMakeLists.txt +++ b/tests/e2e/CMakeLists.txt @@ -23,4 +23,10 @@ add_test (NAME ben_bot.one_shot_clis "$" ) -set_tests_properties (ben_bot.one_shot_clis PROPERTIES REQUIRED_FILES "$") +add_test (NAME ben_bot.e2e COMMAND Python::Interpreter "${CMAKE_CURRENT_LIST_DIR}/e2e.py" + "$" +) + +set_tests_properties ( + ben_bot.one_shot_clis ben_bot.e2e PROPERTIES REQUIRED_FILES "$" +) diff --git a/tests/e2e/e2e.py b/tests/e2e/e2e.py index 2feeb369..a2e9e18f 100644 --- a/tests/e2e/e2e.py +++ b/tests/e2e/e2e.py @@ -27,122 +27,56 @@ def get_path(): - return os.path.abspath(os.path.join(CWD, args.stockfish_path)) + return os.path.abspath(os.path.join(CWD, args.engine_path)) def BenBot(*args, **kwargs): return Engine(get_path(), *args, **kwargs) -class TestCLI(metaclass=OrderedClassMembers): - def test_go_nodes_1000(self): - engine = BenBot("go nodes 1000".split(" "), True) - assert engine.process.returncode == 0 - - def test_go_depth_10(self): - engine = BenBot("go depth 10".split(" "), True) - assert engine.process.returncode == 0 - - def test_go_perft_4(self): - engine = BenBot("perft 4 json".split(" "), True) - assert engine.process.returncode == 0 - - def test_go_movetime_1000(self): - engine = BenBot("go movetime 1000".split(" "), True) - assert engine.process.returncode == 0 - - def test_go_wtime_8000_btime_8000_winc_500_binc_500(self): - engine = BenBot( - "go wtime 8000 btime 8000 winc 500 binc 500".split(" "), - True, - ) - assert engine.process.returncode == 0 - - def test_go_wtime_1000_btime_1000_winc_0_binc_0(self): - engine = BenBot( - "go wtime 1000 btime 1000 winc 0 binc 0".split(" "), - True, - ) - assert engine.process.returncode == 0 - - def test_go_wtime_1000_btime_1000_winc_0_binc_0_movestogo_5(self): - engine = BenBot( - "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5".split(" "), - True, - ) - assert engine.process.returncode == 0 - - def test_go_movetime_200(self): - engine = BenBot("go movetime 200".split(" "), True) - assert engine.process.returncode == 0 - - def test_go_nodes_20000_searchmoves_e2e4_d2d4(self): - engine = BenBot("go nodes 20000 searchmoves e2e4 d2d4".split(" "), True) - assert engine.process.returncode == 0 - - def test_bench(self): - engine = BenBot( - f"bench".split(" "), - True, - ) - assert engine.process.returncode == 0 - - def test_showpos(self): - engine = BenBot("showpos".split(" "), True) - assert engine.process.returncode == 0 - - def test_compiler(self): - engine = BenBot("compiler".split(" "), True) - assert engine.process.returncode == 0 - - def test_uci(self): - engine = BenBot("uci".split(" "), True) - assert engine.process.returncode == 0 - - class TestInteractive(metaclass=OrderedClassMembers): def __init__(self): - self.stockfish = BenBot() + self.engine = BenBot() def afterEach(self): - self.stockfish.clear_output() + self.engine.clear_output() def test_uci_command(self): - self.stockfish.send_command("uci") - self.stockfish.equals("uciok") + self.engine.send_command("uci") + self.engine.equals("uciok") def test_set_threads_option(self): - self.stockfish.send_command(f"setoption name Threads value 1") + self.engine.send_command(f"setoption name Threads value 1") def test_ucinewgame_and_startpos_nodes_1000(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command("position startpos") - self.stockfish.send_command("go nodes 1000") - self.stockfish.starts_with("bestmove") + self.engine.send_command("ucinewgame") + self.engine.send_command("position startpos") + self.engine.send_command("go nodes 1000") + self.engine.starts_with("bestmove") def test_ucinewgame_and_startpos_moves(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command("position startpos moves e2e4 e7e6") - self.stockfish.send_command("go nodes 1000") - self.stockfish.starts_with("bestmove") + self.engine.send_command("ucinewgame") + self.engine.send_command("position startpos moves e2e4 e7e6") + self.engine.send_command("go nodes 1000") + self.engine.starts_with("bestmove") def test_fen_position_1(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") - self.stockfish.send_command("go nodes 1000") - self.stockfish.starts_with("bestmove") + self.engine.send_command("ucinewgame") + self.engine.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") + self.engine.send_command("go nodes 1000") + self.engine.starts_with("bestmove") def test_fen_position_2_flip(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") - self.stockfish.send_command("flip") - self.stockfish.send_command("go nodes 1000") - self.stockfish.starts_with("bestmove") + self.engine.send_command("ucinewgame") + self.engine.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") + self.engine.send_command("flip") + self.engine.send_command("go nodes 1000") + self.engine.starts_with("bestmove") def test_depth_5_with_callback(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command("position startpos") - self.stockfish.send_command("go depth 5") + self.engine.send_command("ucinewgame") + self.engine.send_command("position startpos") + self.engine.send_command("go depth 5") def callback(output): regex = r"info depth \d+ seldepth \d+ score cp -?\d+ time \d+ hashfull \d+ nodes \d+ nps \d+ tbhits \d+ pv" @@ -152,14 +86,14 @@ def callback(output): return True return False - self.stockfish.check_output(callback) + self.engine.check_output(callback) def test_ucinewgame_and_go_depth_4(self): total_depth = 4 - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command("position startpos") - self.stockfish.send_command(f"go depth {total_depth}") + self.engine.send_command("ucinewgame") + self.engine.send_command("position startpos") + self.engine.send_command(f"go depth {total_depth}") depth = 0 @@ -181,209 +115,199 @@ def callback(output): return False - self.stockfish.check_output(callback) + self.engine.check_output(callback) def test_clear_hash(self): - self.stockfish.send_command("setoption name Clear Hash") + self.engine.send_command("setoption name Clear Hash") def test_fen_position_mate_1(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( - "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6" - ) - self.stockfish.send_command("go depth 10") + self.engine.send_command("ucinewgame") + self.engine.send_command("position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6") + self.engine.send_command("go depth 10") - self.stockfish.expect("* score mate 1 * pv d5e6") - self.stockfish.equals("bestmove d5e6") + self.engine.expect("* score mate 1 * pv d5e6") + self.engine.equals("bestmove d5e6") def test_fen_position_mate_minus_1(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( + self.engine.send_command("ucinewgame") + self.engine.send_command( "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -" ) - self.stockfish.send_command("go depth 10") - self.stockfish.expect("* score mate -1 *") - self.stockfish.starts_with("bestmove") + self.engine.send_command("go depth 10") + self.engine.expect("* score mate -1 *") + self.engine.starts_with("bestmove") def test_fen_position_fixed_node(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( + self.engine.send_command("ucinewgame") + self.engine.send_command( "position fen 5K2/8/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1" ) - self.stockfish.send_command("go nodes 10000") - self.stockfish.starts_with("bestmove") + self.engine.send_command("go nodes 10000") + self.engine.starts_with("bestmove") def test_fen_position_with_mate_go_depth(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( - "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -" - ) - self.stockfish.send_command("go depth 18 searchmoves c6d7") - self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5") + self.engine.send_command("ucinewgame") + self.engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") + self.engine.send_command("go depth 18 searchmoves c6d7") + self.engine.expect("* score mate 2 * pv c6d7 * f7f5") - self.stockfish.starts_with("bestmove") + self.engine.starts_with("bestmove") def test_fen_position_with_mate_go_mate(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( - "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -" - ) - self.stockfish.send_command("go mate 2 searchmoves c6d7") - self.stockfish.expect("* score mate 2 * pv c6d7 *") + self.engine.send_command("ucinewgame") + self.engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") + self.engine.send_command("go mate 2 searchmoves c6d7") + self.engine.expect("* score mate 2 * pv c6d7 *") - self.stockfish.starts_with("bestmove") + self.engine.starts_with("bestmove") def test_fen_position_with_mate_go_nodes(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( - "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -" - ) - self.stockfish.send_command("go nodes 500000 searchmoves c6d7") - self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5") + self.engine.send_command("ucinewgame") + self.engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") + self.engine.send_command("go nodes 500000 searchmoves c6d7") + self.engine.expect("* score mate 2 * pv c6d7 * f7f5") - self.stockfish.starts_with("bestmove") + self.engine.starts_with("bestmove") def test_fen_position_depth_8(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( + self.engine.send_command("ucinewgame") + self.engine.send_command( "position fen r1b2r1k/pp1p2pp/2p5/2B1q3/8/8/P1PN2PP/R4RK1 w - - 0 18" ) - self.stockfish.send_command("go depth 8") - self.stockfish.contains("score mate 1") + self.engine.send_command("go depth 8") + self.engine.contains("score mate 1") - self.stockfish.starts_with("bestmove") + self.engine.starts_with("bestmove") def test_fen_position_with_mate_go_depth_and_promotion(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( + self.engine.send_command("ucinewgame") + self.engine.send_command( "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q" ) - self.stockfish.send_command("go depth 13") - self.stockfish.expect("* score mate 1 * pv f7f5") - self.stockfish.starts_with("bestmove f7f5") + self.engine.send_command("go depth 13") + self.engine.expect("* score mate 1 * pv f7f5") + self.engine.starts_with("bestmove f7f5") def test_fen_position_with_mate_go_depth_and_searchmoves(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( - "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -" - ) - self.stockfish.send_command("go depth 18 searchmoves c6d7") - self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5") + self.engine.send_command("ucinewgame") + self.engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") + self.engine.send_command("go depth 18 searchmoves c6d7") + self.engine.expect("* score mate 2 * pv c6d7 * f7f5") - self.stockfish.starts_with("bestmove c6d7") + self.engine.starts_with("bestmove c6d7") def test_fen_position_with_moves_with_mate_go_depth_and_searchmoves(self): - self.stockfish.send_command("ucinewgame") - self.stockfish.send_command( + self.engine.send_command("ucinewgame") + self.engine.send_command( "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7" ) - self.stockfish.send_command("go depth 18 searchmoves e3e2") - self.stockfish.expect("* score mate -1 * pv e3e2 f7f5") - self.stockfish.starts_with("bestmove e3e2") + self.engine.send_command("go depth 18 searchmoves e3e2") + self.engine.expect("* score mate -1 * pv e3e2 f7f5") + self.engine.starts_with("bestmove e3e2") class TestEnPassantSanitization(metaclass=OrderedClassMembers): def __init__(self): - self.stockfish = BenBot() + self.engine = BenBot() def afterEach(self): - self.stockfish.clear_output() + self.engine.clear_output() def test_position_1(self): - self.stockfish.send_command( + self.engine.send_command( "position fen rnbqkbnr/ppp1p1pp/5p2/3pP3/8/8/PPPP1PPP/RNBQKBNR w kq d6 0 3" ) - self.stockfish.send_command("showpos") + self.engine.send_command("showpos") - self.stockfish.expect_for_line_matching( + self.engine.expect_for_line_matching( "FEN*", "*rnbqkbnr/ppp1p1pp/5p2/3pP3/8/8/PPPP1PPP/RNBQKBNR w kq d6 0 3*" ) def test_position_2(self): - self.stockfish.send_command("position fen k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1") - self.stockfish.send_command("showpos") + self.engine.send_command("position fen k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1") + self.engine.send_command("showpos") - self.stockfish.expect_for_line_matching( + self.engine.expect_for_line_matching( "FEN*", "*k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1*" ) # def test_position_3(self): - # self.stockfish.send_command("position fen k1r5/8/8/1pP5/2K5/8/8/8 w - b6 0 1") - # self.stockfish.send_command("showpos") + # self.engine.send_command("position fen k1r5/8/8/1pP5/2K5/8/8/8 w - b6 0 1") + # self.engine.send_command("showpos") # - # self.stockfish.expect_for_line_matching( + # self.engine.expect_for_line_matching( # "XFEN*", "*k1r5/8/8/1pP5/2K5/8/8/8 w - - 0 1*" # ) # def test_position_4(self): - # self.stockfish.send_command("position fen k1r5/8/8/1pP5/8/2K5/8/8 w - b6 0 1") - # self.stockfish.send_command("showpos") + # self.engine.send_command("position fen k1r5/8/8/1pP5/8/2K5/8/8 w - b6 0 1") + # self.engine.send_command("showpos") # - # self.stockfish.expect_for_line_matching( + # self.engine.expect_for_line_matching( # "XFEN*", "*k1r5/8/8/1pP5/8/2K5/8/8 w - - 0 1*" # ) def test_position_5(self): - self.stockfish.send_command("position fen k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1") - self.stockfish.send_command("showpos") + self.engine.send_command("position fen k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1") + self.engine.send_command("showpos") - self.stockfish.expect_for_line_matching( + self.engine.expect_for_line_matching( "FEN*", "*k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1*" ) def test_position_6(self): - self.stockfish.send_command("position fen k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1") - self.stockfish.send_command("showpos") + self.engine.send_command("position fen k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1") + self.engine.send_command("showpos") - self.stockfish.expect_for_line_matching( + self.engine.expect_for_line_matching( "FEN*", "*k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1*" ) def test_position_7(self): - self.stockfish.send_command("position fen k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1") - self.stockfish.send_command("showpos") + self.engine.send_command("position fen k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1") + self.engine.send_command("showpos") - self.stockfish.expect_for_line_matching( + self.engine.expect_for_line_matching( "FEN*", "*k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1*" ) # def test_position_8(self): - # self.stockfish.send_command("position fen k7/b5b1/8/2PpP3/3K4/8/8/8 w - d6 0 1") - # self.stockfish.send_command("showpos") + # self.engine.send_command("position fen k7/b5b1/8/2PpP3/3K4/8/8/8 w - d6 0 1") + # self.engine.send_command("showpos") # - # self.stockfish.expect_for_line_matching( + # self.engine.expect_for_line_matching( # "XFEN*", "*k7/b5b1/8/2PpP3/3K4/8/8/8 w - - 0 1*" # ) # def test_position_9(self): - # self.stockfish.send_command("position fen k7/8/8/r2pPK2/8/8/8/8 w - d6 0 1") - # self.stockfish.send_command("showpos") + # self.engine.send_command("position fen k7/8/8/r2pPK2/8/8/8/8 w - d6 0 1") + # self.engine.send_command("showpos") # - # self.stockfish.expect_for_line_matching( + # self.engine.expect_for_line_matching( # "XFEN*", "*k7/8/8/r2pPK2/8/8/8/8 w - - 0 1*" # ) def test_position_10(self): - self.stockfish.send_command("position fen k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1") - self.stockfish.send_command("showpos") + self.engine.send_command("position fen k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1") + self.engine.send_command("showpos") - self.stockfish.expect_for_line_matching( + self.engine.expect_for_line_matching( "FEN*", "*k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1*" ) def test_position_11(self): - self.stockfish.send_command("position fen kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1") - self.stockfish.send_command("showpos") + self.engine.send_command("position fen kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1") + self.engine.send_command("showpos") - self.stockfish.expect_for_line_matching( + self.engine.expect_for_line_matching( "FEN*", "*kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1*" ) def parse_args(): - parser = argparse.ArgumentParser(description="Run Stockfish with testing options") + parser = argparse.ArgumentParser(description="Run BenBot end-to-end tests") - parser.add_argument("stockfish_path", type=str, help="Path to Stockfish binary") + parser.add_argument("engine_path", type=str, help="Path to BenBot binary") return parser.parse_args() @@ -394,7 +318,7 @@ def parse_args(): framework = MiniTestFramework() # Each test suite will be run inside a temporary directory - framework.run([TestCLI, TestInteractive, TestEnPassantSanitization]) + framework.run([TestInteractive, TestEnPassantSanitization]) if framework.has_failed(): sys.exit(1) diff --git a/tests/e2e/testing.py b/tests/e2e/testing.py index 1cc68126..e71cf177 100644 --- a/tests/e2e/testing.py +++ b/tests/e2e/testing.py @@ -11,7 +11,6 @@ # ====================================================================================== import subprocess -from subprocess import CompletedProcess from typing import List import os import collections @@ -237,7 +236,7 @@ def __del__(self): def _check_process_alive(self): if not self.process or self.process.poll() is not None: print("\n".join(self.output)) - raise RuntimeError("Stockfish process has terminated") + raise RuntimeError("BenBot process has terminated") def start(self): if self.cli: @@ -307,7 +306,7 @@ def check_output(self, callback): raise ValueError("Callback function is required") for line in self.readline(): - if callback(line) == True: + if callback(line): return @timeout_decorator(MAX_TIMEOUT) From 987c10627cec81ce77713b2deeed37af999601fb Mon Sep 17 00:00:00 2001 From: Ben Vining Date: Sun, 15 Mar 2026 21:56:20 -0500 Subject: [PATCH 6/6] refactor: e2e test script --- tests/e2e/e2e.py | 372 +++++++++++++++++++++++-------------------- tests/e2e/testing.py | 30 +--- 2 files changed, 199 insertions(+), 203 deletions(-) diff --git a/tests/e2e/e2e.py b/tests/e2e/e2e.py index a2e9e18f..7042e87c 100644 --- a/tests/e2e/e2e.py +++ b/tests/e2e/e2e.py @@ -35,48 +35,56 @@ def BenBot(*args, **kwargs): class TestInteractive(metaclass=OrderedClassMembers): - def __init__(self): - self.engine = BenBot() - - def afterEach(self): - self.engine.clear_output() - - def test_uci_command(self): - self.engine.send_command("uci") - self.engine.equals("uciok") - - def test_set_threads_option(self): - self.engine.send_command(f"setoption name Threads value 1") - - def test_ucinewgame_and_startpos_nodes_1000(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position startpos") - self.engine.send_command("go nodes 1000") - self.engine.starts_with("bestmove") - - def test_ucinewgame_and_startpos_moves(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position startpos moves e2e4 e7e6") - self.engine.send_command("go nodes 1000") - self.engine.starts_with("bestmove") - - def test_fen_position_1(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") - self.engine.send_command("go nodes 1000") - self.engine.starts_with("bestmove") - - def test_fen_position_2_flip(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") - self.engine.send_command("flip") - self.engine.send_command("go nodes 1000") - self.engine.starts_with("bestmove") - - def test_depth_5_with_callback(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position startpos") - self.engine.send_command("go depth 5") + @staticmethod + def test_uci_command(): + engine = BenBot() + engine.send_command("uci") + engine.equals("uciok") + + @staticmethod + def test_set_threads_option(): + engine = BenBot() + engine.send_command(f"setoption name Threads value 1") + + @staticmethod + def test_ucinewgame_and_startpos_nodes_1000(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position startpos") + engine.send_command("go nodes 1000") + engine.starts_with("bestmove") + + @staticmethod + def test_ucinewgame_and_startpos_moves(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position startpos moves e2e4 e7e6") + engine.send_command("go nodes 1000") + engine.starts_with("bestmove") + + @staticmethod + def test_fen_position_1(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") + engine.send_command("go nodes 1000") + engine.starts_with("bestmove") + + @staticmethod + def test_fen_position_2_flip(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1") + engine.send_command("flip") + engine.send_command("go nodes 1000") + engine.starts_with("bestmove") + + @staticmethod + def test_depth_5_with_callback(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position startpos") + engine.send_command("go depth 5") def callback(output): regex = r"info depth \d+ seldepth \d+ score cp -?\d+ time \d+ hashfull \d+ nodes \d+ nps \d+ tbhits \d+ pv" @@ -86,14 +94,16 @@ def callback(output): return True return False - self.engine.check_output(callback) + engine.check_output(callback) - def test_ucinewgame_and_go_depth_4(self): + @staticmethod + def test_ucinewgame_and_go_depth_4(): total_depth = 4 - self.engine.send_command("ucinewgame") - self.engine.send_command("position startpos") - self.engine.send_command(f"go depth {total_depth}") + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position startpos") + engine.send_command(f"go depth {total_depth}") depth = 0 @@ -115,121 +125,135 @@ def callback(output): return False - self.engine.check_output(callback) - - def test_clear_hash(self): - self.engine.send_command("setoption name Clear Hash") - - def test_fen_position_mate_1(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6") - self.engine.send_command("go depth 10") - - self.engine.expect("* score mate 1 * pv d5e6") - self.engine.equals("bestmove d5e6") - - def test_fen_position_mate_minus_1(self): - self.engine.send_command("ucinewgame") - self.engine.send_command( - "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -" - ) - self.engine.send_command("go depth 10") - self.engine.expect("* score mate -1 *") - self.engine.starts_with("bestmove") - - def test_fen_position_fixed_node(self): - self.engine.send_command("ucinewgame") - self.engine.send_command( - "position fen 5K2/8/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1" - ) - self.engine.send_command("go nodes 10000") - self.engine.starts_with("bestmove") - - def test_fen_position_with_mate_go_depth(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") - self.engine.send_command("go depth 18 searchmoves c6d7") - self.engine.expect("* score mate 2 * pv c6d7 * f7f5") - - self.engine.starts_with("bestmove") - - def test_fen_position_with_mate_go_mate(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") - self.engine.send_command("go mate 2 searchmoves c6d7") - self.engine.expect("* score mate 2 * pv c6d7 *") - - self.engine.starts_with("bestmove") - - def test_fen_position_with_mate_go_nodes(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") - self.engine.send_command("go nodes 500000 searchmoves c6d7") - self.engine.expect("* score mate 2 * pv c6d7 * f7f5") - - self.engine.starts_with("bestmove") - - def test_fen_position_depth_8(self): - self.engine.send_command("ucinewgame") - self.engine.send_command( + engine.check_output(callback) + + @staticmethod + def test_clear_hash(): + engine = BenBot() + engine.send_command("setoption name Clear Hash") + + @staticmethod + def test_fen_position_mate_1(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6") + engine.send_command("go depth 10") + + engine.expect("* score mate 1 * pv d5e6") + engine.equals("bestmove d5e6") + + @staticmethod + def test_fen_position_mate_minus_1(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -") + engine.send_command("go depth 10") + engine.expect("* score mate -1 *") + engine.starts_with("bestmove") + + @staticmethod + def test_fen_position_fixed_node(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 5K2/8/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1") + engine.send_command("go nodes 10000") + engine.starts_with("bestmove") + + @staticmethod + def test_fen_position_with_mate_go_depth(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") + engine.send_command("go depth 18 searchmoves c6d7") + engine.expect("* score mate 2 * pv c6d7 * f7f5") + + engine.starts_with("bestmove") + + @staticmethod + def test_fen_position_with_mate_go_mate(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") + engine.send_command("go mate 2 searchmoves c6d7") + engine.expect("* score mate 2 * pv c6d7 *") + + engine.starts_with("bestmove") + + @staticmethod + def test_fen_position_with_mate_go_nodes(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") + engine.send_command("go nodes 500000 searchmoves c6d7") + engine.expect("* score mate 2 * pv c6d7 * f7f5") + + engine.starts_with("bestmove") + + @staticmethod + def test_fen_position_depth_8(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command( "position fen r1b2r1k/pp1p2pp/2p5/2B1q3/8/8/P1PN2PP/R4RK1 w - - 0 18" ) - self.engine.send_command("go depth 8") - self.engine.contains("score mate 1") + engine.send_command("go depth 8") + engine.contains("score mate 1") - self.engine.starts_with("bestmove") + engine.starts_with("bestmove") - def test_fen_position_with_mate_go_depth_and_promotion(self): - self.engine.send_command("ucinewgame") - self.engine.send_command( + @staticmethod + def test_fen_position_with_mate_go_depth_and_promotion(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command( "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q" ) - self.engine.send_command("go depth 13") - self.engine.expect("* score mate 1 * pv f7f5") - self.engine.starts_with("bestmove f7f5") - - def test_fen_position_with_mate_go_depth_and_searchmoves(self): - self.engine.send_command("ucinewgame") - self.engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") - self.engine.send_command("go depth 18 searchmoves c6d7") - self.engine.expect("* score mate 2 * pv c6d7 * f7f5") - - self.engine.starts_with("bestmove c6d7") - - def test_fen_position_with_moves_with_mate_go_depth_and_searchmoves(self): - self.engine.send_command("ucinewgame") - self.engine.send_command( + engine.send_command("go depth 13") + engine.expect("* score mate 1 * pv f7f5") + engine.starts_with("bestmove f7f5") + + @staticmethod + def test_fen_position_with_mate_go_depth_and_searchmoves(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command("position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -") + engine.send_command("go depth 18 searchmoves c6d7") + engine.expect("* score mate 2 * pv c6d7 * f7f5") + + engine.starts_with("bestmove c6d7") + + @staticmethod + def test_fen_position_with_moves_with_mate_go_depth_and_searchmoves(): + engine = BenBot() + engine.send_command("ucinewgame") + engine.send_command( "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7" ) - self.engine.send_command("go depth 18 searchmoves e3e2") - self.engine.expect("* score mate -1 * pv e3e2 f7f5") - self.engine.starts_with("bestmove e3e2") + engine.send_command("go depth 18 searchmoves e3e2") + engine.expect("* score mate -1 * pv e3e2 f7f5") + engine.starts_with("bestmove e3e2") class TestEnPassantSanitization(metaclass=OrderedClassMembers): - def __init__(self): - self.engine = BenBot() - - def afterEach(self): - self.engine.clear_output() - - def test_position_1(self): - self.engine.send_command( + @staticmethod + def test_position_1(): + engine = BenBot() + engine.send_command( "position fen rnbqkbnr/ppp1p1pp/5p2/3pP3/8/8/PPPP1PPP/RNBQKBNR w kq d6 0 3" ) - self.engine.send_command("showpos") + engine.send_command("showpos") - self.engine.expect_for_line_matching( + engine.expect_for_line_matching( "FEN*", "*rnbqkbnr/ppp1p1pp/5p2/3pP3/8/8/PPPP1PPP/RNBQKBNR w kq d6 0 3*" ) - def test_position_2(self): - self.engine.send_command("position fen k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1") - self.engine.send_command("showpos") + @staticmethod + def test_position_2(): + engine = BenBot() + engine.send_command("position fen k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1") + engine.send_command("showpos") - self.engine.expect_for_line_matching( - "FEN*", "*k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1*" - ) + engine.expect_for_line_matching("FEN*", "*k7/8/8/1pP5/2K5/8/8/8 w - b6 0 1*") # def test_position_3(self): # self.engine.send_command("position fen k1r5/8/8/1pP5/2K5/8/8/8 w - b6 0 1") @@ -247,29 +271,29 @@ def test_position_2(self): # "XFEN*", "*k1r5/8/8/1pP5/8/2K5/8/8 w - - 0 1*" # ) - def test_position_5(self): - self.engine.send_command("position fen k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1") - self.engine.send_command("showpos") + @staticmethod + def test_position_5(): + engine = BenBot() + engine.send_command("position fen k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1") + engine.send_command("showpos") - self.engine.expect_for_line_matching( - "FEN*", "*k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1*" - ) + engine.expect_for_line_matching("FEN*", "*k1r5/8/8/PpP5/8/2K5/8/8 w - b6 0 1*") - def test_position_6(self): - self.engine.send_command("position fen k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1") - self.engine.send_command("showpos") + @staticmethod + def test_position_6(): + engine = BenBot() + engine.send_command("position fen k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1") + engine.send_command("showpos") - self.engine.expect_for_line_matching( - "FEN*", "*k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1*" - ) + engine.expect_for_line_matching("FEN*", "*k1r5/8/8/PpP5/2K5/8/8/8 w - b6 0 1*") - def test_position_7(self): - self.engine.send_command("position fen k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1") - self.engine.send_command("showpos") + @staticmethod + def test_position_7(): + engine = BenBot() + engine.send_command("position fen k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1") + engine.send_command("showpos") - self.engine.expect_for_line_matching( - "FEN*", "*k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1*" - ) + engine.expect_for_line_matching("FEN*", "*k7/4b3/8/PpP5/1K6/8/8/8 w - b6 0 1*") # def test_position_8(self): # self.engine.send_command("position fen k7/b5b1/8/2PpP3/3K4/8/8/8 w - d6 0 1") @@ -287,21 +311,21 @@ def test_position_7(self): # "XFEN*", "*k7/8/8/r2pPK2/8/8/8/8 w - - 0 1*" # ) - def test_position_10(self): - self.engine.send_command("position fen k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1") - self.engine.send_command("showpos") + @staticmethod + def test_position_10(): + engine = BenBot() + engine.send_command("position fen k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1") + engine.send_command("showpos") - self.engine.expect_for_line_matching( - "FEN*", "*k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1*" - ) + engine.expect_for_line_matching("FEN*", "*k7/8/8/r1PpPK2/8/8/8/8 w - d6 0 1*") - def test_position_11(self): - self.engine.send_command("position fen kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1") - self.engine.send_command("showpos") + @staticmethod + def test_position_11(): + engine = BenBot() + engine.send_command("position fen kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1") + engine.send_command("showpos") - self.engine.expect_for_line_matching( - "FEN*", "*kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1*" - ) + engine.expect_for_line_matching("FEN*", "*kb6/8/8/3pP3/5K2/8/8/8 w - d6 0 1*") def parse_args(): diff --git a/tests/e2e/testing.py b/tests/e2e/testing.py index e71cf177..cb566301 100644 --- a/tests/e2e/testing.py +++ b/tests/e2e/testing.py @@ -138,14 +138,8 @@ def __run_test_method(self, test_instance, method: str) -> int: t0 = time.time() with redirect_stdout(buffer): - if hasattr(test_instance, "beforeEach"): - test_instance.beforeEach() - getattr(test_instance, method)() - if hasattr(test_instance, "afterEach"): - test_instance.afterEach() - duration = time.time() - t0 self.print_success(f" {method} ({duration * 1000:.2f}ms)") @@ -215,16 +209,10 @@ def print_success(add: str): class BenBot: - def __init__( - self, - path: str, - args: List[str] = [], - cli: bool = False, - ): + def __init__(self, path: str, args: List[str] = []): self.path = path self.process = None self.args = args - self.cli = cli self.output = [] self.start() @@ -239,22 +227,6 @@ def _check_process_alive(self): raise RuntimeError("BenBot process has terminated") def start(self): - if self.cli: - self.process = subprocess.run( - [self.path] + self.args + ["--no-loop"], - capture_output=True, - text=True, - ) - - if self.process.returncode != 0: - print(self.process.stdout) - print(self.process.stderr) - print(f"Process failed with return code {self.process.returncode}") - - self.process = None - - return - self.process = subprocess.Popen( [self.path] + self.args, stdin=subprocess.PIPE,