Improve github actions setup

.github/workflows/build.yml

@@ -11,29 +11,50 @@ on:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
-  cancel-in-progress: true
+  # group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-${{ matrix.os }}-${{ matrix.type }}
+  # cancel-in-progress: true
 
 jobs:
   build_and_test:
     strategy:
+      fail-fast: false
       matrix:
-        os: [macos-12, macos-13, macos-14, macos-latest, ubuntu-22.04, ubuntu-latest, windows-2019, windows-latest]
+        os: [
+          # macos-13,
+          # macos-14,
+          macos-latest,
+          # ubuntu-22.04,
+          ubuntu-latest,
+          # windows-2019,
+          windows-latest,
+        ]
+        type: [
+          Release,
+          # Debug,
+        ]
     runs-on:  ${{ matrix.os }}
+    timeout-minutes: 30
 
     steps:
       - name: Clone
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2.11
+        with:
+          key: ${{ matrix.os }}-${{ matrix.type }}
+
       - name: Set up CMake
         uses: lukka/get-cmake@latest
 
       - name: Configure CMake
-        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=Release
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 
       - name: Build
         run: cmake --build ${{github.workspace}}/build --config Release --parallel
 
       - name: Test
-        run: ctest --test-dir build/tests --verbose
+        working-directory: ${{github.workspace}}/build
+        run: ctest --test-dir tests --output-on-failure --verbose

CMakeLists.txt

@@ -7,10 +7,21 @@
 # SPDX-License-Identifier: MIT
 cmake_minimum_required(VERSION 3.14)
 
+cmake_policy(SET CMP0135 NEW) # https://cmake.org/cmake/help/latest/policy/CMP0135.html
+
 project(minja VERSION 1.0.0 LANGUAGES CXX)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
+option(MINJA_TEST_GATED_MODELS    "minja: test gated models"    OFF)
+if (MSVC)
+    set(MINJA_FUZZTEST_ENABLED_DEFAULT OFF)
+else()
+    set(MINJA_FUZZTEST_ENABLED_DEFAULT ON)
+endif()
+option(MINJA_FUZZTEST_ENABLED       "minja: fuzztests enabled"  MINJA_FUZZTEST_ENABLED_DEFAULT)
+option(MINJA_FUZZTEST_FUZZING_MODE  "minja: run fuzztests (if enabled) in fuzzing mode"     OFF)
+
 # Note: tests require C++14 because google/fuzztest depends on a version of gtest that requires it
 # (and we don't want to use an older version of fuzztest)
 # Examples are built w/ C++11 to check the compatibility of the library.
@@ -22,9 +33,15 @@ include(FetchContent)
 FetchContent_Declare(json URL https://github.com/nlohmann/json/archive/refs/heads/develop.zip)
 FetchContent_MakeAvailable(json)
 
-# Fetch google/fuzztest (and indirectly, gtest)
-FetchContent_Declare(fuzztest URL https://github.com/google/fuzztest/archive/refs/heads/main.zip)
-FetchContent_MakeAvailable(fuzztest)
+if (MINJA_FUZZTEST_ENABLED)
+    # Fetch google/fuzztest (and indirectly, gtest)
+    FetchContent_Declare(fuzztest URL https://github.com/google/fuzztest/archive/refs/heads/main.zip)
+    FetchContent_MakeAvailable(fuzztest)
+else()
+    # Fetch gtest
+    FetchContent_Declare(googletest URL https://github.com/google/googletest/archive/refs/heads/main.zip)
+    FetchContent_MakeAvailable(googletest)
+endif()
 
 # Use ccache if installed
 find_program(CCACHE_PATH ccache)
@@ -40,41 +57,21 @@ if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
 endif()
 
 # Create a python venv w/ the required dependencies
-find_package(Python COMPONENTS Interpreter REQUIRED)
-function(setup_python_environment)
-    set(VENV_DIR "${CMAKE_BINARY_DIR}/venv")
-
-    # Create a virtual environment if it doesnt exist yet
-    if(EXISTS ${VENV_DIR})
-        set(Python_EXECUTABLE ${VENV_DIR}/bin/python PARENT_SCOPE)
-        return()
-    endif()
-
-    execute_process(
-        COMMAND ${Python_EXECUTABLE} -m venv ${VENV_DIR}
-        RESULT_VARIABLE VENV_RESULT
-    )
-    if(NOT VENV_RESULT EQUAL 0)
-        message(FATAL_ERROR "Failed to create virtual environment")
-    endif()
-
-    if(WIN32)
-        set(VENV_PYTHON "${VENV_DIR}/Scripts/python.exe")
-    else()
-        set(VENV_PYTHON "${VENV_DIR}/bin/python")
-    endif()
-
-    execute_process(
-        COMMAND ${VENV_PYTHON} -m pip install -r "${CMAKE_SOURCE_DIR}/requirements.txt"
-        RESULT_VARIABLE PIP_RESULT
-    )
-    if(NOT PIP_RESULT EQUAL 0)
-        message(FATAL_ERROR "Failed to install Python dependencies")
-    endif()
-
-    set(Python_EXECUTABLE ${VENV_PYTHON} PARENT_SCOPE)
-endfunction()
-setup_python_environment()
+find_package(Python3 COMPONENTS Interpreter REQUIRED)
+set(VENV_DIR "${CMAKE_BINARY_DIR}/venv")
+if(WIN32)
+    set(VENV_PYTHON "${VENV_DIR}/Scripts/python.exe")
+else()
+    set(VENV_PYTHON "${VENV_DIR}/bin/python")
+endif()
+execute_process(
+    COMMAND ${Python3_EXECUTABLE} -m venv "${VENV_DIR}"
+    COMMAND_ERROR_IS_FATAL ANY)
+execute_process(
+    COMMAND ${VENV_PYTHON} -m pip install -r "${CMAKE_SOURCE_DIR}/requirements.txt"
+    COMMAND_ERROR_IS_FATAL ANY)
+set(Python_EXECUTABLE "${VENV_PYTHON}" CACHE FILEPATH "Path to Python executable in venv" FORCE)
+message(STATUS "Python executable: ${Python_EXECUTABLE}")
 
 find_program(CPPCHECK cppcheck)
 if(CPPCHECK)
@@ -92,4 +89,5 @@ include_directories(
 add_subdirectory(examples)
 
 enable_testing()
+include(GoogleTest)
 add_subdirectory(tests)

README.md

@@ -199,7 +199,7 @@ Main limitations (non-exhaustive list):
     - Which version of GCC / clang did you compile the tests with? On which OS version?
     - If you intend to contribute a fix:
         - Please read [CONTRIBUTING](./CONTRIBUTING.md) first. You'd have to sign a CLA, which your employer may need to accept.
-        - Please test as many gated models as possible
+        - Please test as many gated models as possible (use `cmake -B build -DMINJA_TEST_GATED_MODELS=1 ...` and edit `MODEL_IDS` in [tests/CMakeLists.txt](./tests/CMakeLists.txt) appropriately if you don't have access to some models)
 
 - For bonus points, check the style of your edits with:
 

scripts/run_tests.sh

@@ -13,6 +13,6 @@
 #
 set -euo pipefail
 
-cmake -B build && \
+cmake -B build "$@" && \
     cmake --build build -j && \
     ctest --test-dir build -j --output-on-failure

tests/CMakeLists.txt

@@ -50,18 +50,24 @@ set(MODEL_IDS
     "Qwen/Qwen2.5-Math-7B-Instruct"
     "teknium/OpenHermes-2.5-Mistral-7B"
     "TheBloke/FusionNet_34Bx2_MoE-AWQ"
-
-    # Gated models: you will need to run `huggingface-cli login` (and be granted access) to download these
-    #   "meta-llama/Llama-3.2-3B-Instruct"
-    #   "meta-llama/Meta-Llama-3.1-8B-Instruct"
-    #   "google/gemma-7b-it"
-    #   "google/gemma-2-2b-it"
-    #   "mistralai/Mistral-7B-Instruct-v0.2"
-    #   "mistralai/Mixtral-8x7B-Instruct-v0.1"
-    #   "mistralai/Mistral-Nemo-Instruct-2407",
-    #   "CohereForAI/c4ai-command-r-plus"
 )
 
+# Gated models: you will need to run `huggingface-cli login` (and be granted access) to download these
+if (MINJA_TEST_GATED_MODELS)
+    list(APPEND MODEL_IDS
+        "meta-llama/Llama-3.2-3B-Instruct"
+        "meta-llama/Meta-Llama-3.1-8B-Instruct"
+        "google/gemma-7b-it"
+        "google/gemma-2-2b-it"
+        "mistralai/Mistral-7B-Instruct-v0.2"
+        "mistralai/Mixtral-8x7B-Instruct-v0.1"
+        "mistralai/Mistral-Large-Instruct-2407"
+        "mistralai/Mistral-Large-Instruct-2411"
+        "mistralai/Mistral-Nemo-Instruct-2407"
+        "CohereForAI/c4ai-command-r-plus"
+    )
+endif()
+
 # Create one test case for each {template, context} combination
 file(GLOB CONTEXT_FILES "${CMAKE_SOURCE_DIR}/tests/contexts/*.json")
 execute_process(
@@ -81,13 +87,15 @@ foreach(test_case ${CHAT_TEMPLATE_TEST_CASES})
     add_test(NAME ${test_name} COMMAND $<TARGET_FILE:test-chat-template> ${test_args})
 endforeach()
 
-if (FUZZTEST_FUZZING_MODE)
-    message(STATUS "Fuzzing mode enabled")
-    fuzztest_setup_fuzzing_flags()
+if (MINJA_FUZZTEST_ENABLED)
+    if (MINJA_FUZZTEST_FUZZING_MODE)
+        message(STATUS "Fuzzing mode enabled")
+        fuzztest_setup_fuzzing_flags()
+    endif()
+    add_executable(test-fuzz test-fuzz.cpp)
+    set_target_properties(test-fuzz PROPERTIES CXX_STANDARD 17)
+    target_include_directories(test-fuzz PRIVATE ${fuzztest_BINARY_DIR})
+    target_link_libraries(test-fuzz PRIVATE nlohmann_json::nlohmann_json)
+    link_fuzztest(test-fuzz)
+    gtest_discover_tests(test-fuzz)
 endif()
-add_executable(test-fuzz test-fuzz.cpp)
-set_target_properties(test-fuzz PROPERTIES CXX_STANDARD 17)
-target_include_directories(test-fuzz PRIVATE ${fuzztest_BINARY_DIR})
-target_link_libraries(test-fuzz PRIVATE nlohmann_json::nlohmann_json)
-link_fuzztest(test-fuzz)
-gtest_discover_tests(test-fuzz)