test: scripted end-to-end tests

libbenbot/src/engine/Printing.cpp

@@ -151,14 +151,15 @@ void Engine::print_current_position(const string_view arguments) const
 
     println("");
 
-    print_labeled_info("FEN: ", chess::notation::to_fen(pos));
+    print_labeled_info("FEN:  ", chess::notation::to_fen(pos));
+    print_labeled_info("XFEN: ", chess::notation::to_fen(pos, false));
 
     print_labeled_info("Zobrist key: ", std::format("{}", pos.hash));
-
     print_labeled_info("Static eval: ", std::format("{}", eval::evaluate(pos)));
 
     searcher.context.probe_transposition_table(pos)
         .transform([this](const TTData& data) {
+            println("");
             print_labeled_info(
                 "TT hit: ",
                 std::format(

libchess/include/libchess/uci/EngineBase.hpp

@@ -263,6 +263,7 @@ struct EngineBase {
     void handle_quit();
     void handle_setpos(string_view arguments);
     void handle_setoption(string_view arguments);
+    void handle_go(string_view arguments);
 
     static_assert(
         std::atomic_bool::is_always_lock_free,
@@ -313,7 +314,7 @@ struct EngineBase {
             .argsHelp    = "[startpos|fen <fen>] [moves <move...>]" },
         EngineCommand {
             .name   = "go",
-            .action = [this](const string_view args) { go(parse_go_options(args, position)); },
+            .action = [this](const string_view args) { handle_go(args); },
             .description = "Start a search",
             .argsHelp    = { } },
         EngineCommand {

libchess/src/game/Position.cpp

@@ -363,6 +363,8 @@ auto Position::is_illegal() const -> std::optional<std::string>
             magic_enum::enum_name(sideToMove));
     }
 
+    // TODO: our king cannot be in check by more than 2 pieces
+
     return std::nullopt;
 }
 

libchess/src/uci/EngineBase.cpp

@@ -143,6 +143,18 @@ void EngineBase::respond_to_newgame()
     new_game(not wasInitialized);
 }
 
+void EngineBase::handle_go(const string_view arguments)
+{
+    // if we haven't received a ucinewgame yet, make sure the subclass
+    // gets its expected new_game() call before its first go() call
+    if (not initialized.exchange(true, memory_order_relaxed)) {
+        new_game(true);
+    }
+
+    go(
+        parse_go_options(arguments, position));
+}
+
 void EngineBase::handle_quit()
 {
     abort_search();

tests/CMakeLists.txt

@@ -38,6 +38,7 @@ endfunction ()
 
 configure_file (CTestCustom.cmake "${BenBot_BINARY_DIR}/CTestCustom.cmake" @ONLY)
 
+add_subdirectory (e2e)
 add_subdirectory (fastchess)
 add_subdirectory (perft)
 add_subdirectory (position-solver)

tests/e2e/CMakeLists.txt

@@ -0,0 +1,32 @@
+# ======================================================================================
+#
+# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░       ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░      ░▒▓███████▓▒░ ░▒▓██████▓▒░  ░▒▓█▓▒░
+#
+# ======================================================================================
+
+add_feature_info (
+    benbot_e2e_testing "TARGET Python::Interpreter" "BenBot end-to-end & integration testing"
+)
+
+if (NOT TARGET Python::Interpreter)
+    return ()
+endif ()
+
+add_test (NAME ben_bot.one_shot_clis
+          COMMAND Python::Interpreter "${CMAKE_CURRENT_LIST_DIR}/OneShotCLIs.py"
+                  "$<TARGET_FILE:ben_bot>"
+)
+
+add_test (NAME ben_bot.e2e COMMAND Python::Interpreter "${CMAKE_CURRENT_LIST_DIR}/e2e.py"
+                                   "$<TARGET_FILE:ben_bot>"
+)
+
+set_tests_properties (
+    ben_bot.one_shot_clis ben_bot.e2e PROPERTIES REQUIRED_FILES "$<TARGET_FILE:ben_bot>"
+)

tests/e2e/OneShotCLIs.py

@@ -0,0 +1,67 @@
+# ======================================================================================
+#
+# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓███████▓▒░       ░▒▓███████▓▒░ ░▒▓██████▓▒░▒▓████████▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓███████▓▒░░▒▓██████▓▒░ ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓███████▓▒░░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░      ░▒▓█▓▒░░▒▓█▓▒░▒▓█▓▒░░▒▓█▓▒░ ░▒▓█▓▒░
+# ░▒▓███████▓▒░░▒▓████████▓▒░▒▓█▓▒░░▒▓█▓▒░      ░▒▓███████▓▒░ ░▒▓██████▓▒░  ░▒▓█▓▒░
+#
+# ======================================================================================
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def run_oneshot_cli_test_suite(commands: list[str], benbot_path: Path):
+    num_passed = 0
+    num_failed = 0
+
+    def run_benbot_command(command: str):
+        nonlocal num_passed
+        nonlocal num_failed
+
+        print(f"Running command: '{command}'")
+
+        process = subprocess.run(
+            [benbot_path] + command.split(" ") + ["--no-loop"],
+            capture_output=True,
+            text=True,
+        )
+
+        if process.returncode == 0:
+            num_passed += 1
+            return
+
+        print(process.stdout)
+        print(process.stderr)
+        print(f"Process failed with return code {process.returncode}")
+        num_failed += 1
+
+    for cmd in commands:
+        run_benbot_command(cmd)
+
+    print(f"{num_passed} tests passed")
+    print(f"{num_failed} tests failed")
+
+    exit(num_failed)
+
+
+TEST_COMMANDS = [
+    "go nodes 1000",
+    "go depth 10",
+    "perft 4 json",
+    "go movetime 1000",
+    "go wtime 8000 btime 8000 winc 500 binc 500",
+    "go wtime 1000 btime 1000 winc 0 binc 0",
+    "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5",
+    "go movetime 200",
+    "go nodes 20000 searchmoves e2e4 d2d4",
+    "showpos",
+    "compiler",
+    "uci",
+]
+
+run_oneshot_cli_test_suite(TEST_COMMANDS, Path(sys.argv[1]))

tests/e2e/README.md

@@ -0,0 +1,4 @@
+# End-to-end testing
+
+This directory defines tests that are scripted to test the engine
+executable in various scenarios.