diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5093635..0b45bd8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,14 +11,29 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} - cancel-in-progress: true + # group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-${{ matrix.os }}-${{ matrix.type }} + # cancel-in-progress: true jobs: build_and_test: strategy: + fail-fast: false matrix: - os: [macos-12, macos-13, macos-14, macos-latest, ubuntu-22.04, ubuntu-latest, windows-2019, windows-latest] + os: [ + # macos-13, + # macos-14, + macos-latest, + # ubuntu-22.04, + ubuntu-latest, + # windows-2019, + windows-latest, + ] + type: [ + Release, + # Debug, + ] runs-on: ${{ matrix.os }} + timeout-minutes: 30 steps: - name: Clone @@ -26,14 +41,20 @@ jobs: with: fetch-depth: 0 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2.11 + with: + key: ${{ matrix.os }}-${{ matrix.type }} + - name: Set up CMake uses: lukka/get-cmake@latest - name: Configure CMake - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=Release + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - name: Build run: cmake --build ${{github.workspace}}/build --config Release --parallel - name: Test - run: ctest --test-dir build/tests --verbose + working-directory: ${{github.workspace}}/build + run: ctest --test-dir tests --output-on-failure --verbose diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d18166..9b19d4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,10 +7,21 @@ # SPDX-License-Identifier: MIT cmake_minimum_required(VERSION 3.14) +cmake_policy(SET CMP0135 NEW) # https://cmake.org/cmake/help/latest/policy/CMP0135.html + project(minja VERSION 1.0.0 LANGUAGES CXX) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +option(MINJA_TEST_GATED_MODELS "minja: test gated models" OFF) +if (MSVC) + set(MINJA_FUZZTEST_ENABLED_DEFAULT OFF) +else() + set(MINJA_FUZZTEST_ENABLED_DEFAULT ON) +endif() +option(MINJA_FUZZTEST_ENABLED "minja: fuzztests enabled" MINJA_FUZZTEST_ENABLED_DEFAULT) +option(MINJA_FUZZTEST_FUZZING_MODE "minja: run fuzztests (if enabled) in fuzzing mode" OFF) + # Note: tests require C++14 because google/fuzztest depends on a version of gtest that requires it # (and we don't want to use an older version of fuzztest) # Examples are built w/ C++11 to check the compatibility of the library. @@ -22,9 +33,15 @@ include(FetchContent) FetchContent_Declare(json URL https://github.com/nlohmann/json/archive/refs/heads/develop.zip) FetchContent_MakeAvailable(json) -# Fetch google/fuzztest (and indirectly, gtest) -FetchContent_Declare(fuzztest URL https://github.com/google/fuzztest/archive/refs/heads/main.zip) -FetchContent_MakeAvailable(fuzztest) +if (MINJA_FUZZTEST_ENABLED) + # Fetch google/fuzztest (and indirectly, gtest) + FetchContent_Declare(fuzztest URL https://github.com/google/fuzztest/archive/refs/heads/main.zip) + FetchContent_MakeAvailable(fuzztest) +else() + # Fetch gtest + FetchContent_Declare(googletest URL https://github.com/google/googletest/archive/refs/heads/main.zip) + FetchContent_MakeAvailable(googletest) +endif() # Use ccache if installed find_program(CCACHE_PATH ccache) @@ -40,41 +57,21 @@ if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE) endif() # Create a python venv w/ the required dependencies -find_package(Python COMPONENTS Interpreter REQUIRED) -function(setup_python_environment) - set(VENV_DIR "${CMAKE_BINARY_DIR}/venv") - - # Create a virtual environment if it doesnt exist yet - if(EXISTS ${VENV_DIR}) - set(Python_EXECUTABLE ${VENV_DIR}/bin/python PARENT_SCOPE) - return() - endif() - - execute_process( - COMMAND ${Python_EXECUTABLE} -m venv ${VENV_DIR} - RESULT_VARIABLE VENV_RESULT - ) - if(NOT VENV_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to create virtual environment") - endif() - - if(WIN32) - set(VENV_PYTHON "${VENV_DIR}/Scripts/python.exe") - else() - set(VENV_PYTHON "${VENV_DIR}/bin/python") - endif() - - execute_process( - COMMAND ${VENV_PYTHON} -m pip install -r "${CMAKE_SOURCE_DIR}/requirements.txt" - RESULT_VARIABLE PIP_RESULT - ) - if(NOT PIP_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to install Python dependencies") - endif() - - set(Python_EXECUTABLE ${VENV_PYTHON} PARENT_SCOPE) -endfunction() -setup_python_environment() +find_package(Python3 COMPONENTS Interpreter REQUIRED) +set(VENV_DIR "${CMAKE_BINARY_DIR}/venv") +if(WIN32) + set(VENV_PYTHON "${VENV_DIR}/Scripts/python.exe") +else() + set(VENV_PYTHON "${VENV_DIR}/bin/python") +endif() +execute_process( + COMMAND ${Python3_EXECUTABLE} -m venv "${VENV_DIR}" + COMMAND_ERROR_IS_FATAL ANY) +execute_process( + COMMAND ${VENV_PYTHON} -m pip install -r "${CMAKE_SOURCE_DIR}/requirements.txt" + COMMAND_ERROR_IS_FATAL ANY) +set(Python_EXECUTABLE "${VENV_PYTHON}" CACHE FILEPATH "Path to Python executable in venv" FORCE) +message(STATUS "Python executable: ${Python_EXECUTABLE}") find_program(CPPCHECK cppcheck) if(CPPCHECK) @@ -92,4 +89,5 @@ include_directories( add_subdirectory(examples) enable_testing() +include(GoogleTest) add_subdirectory(tests) diff --git a/README.md b/README.md index 3f2ceed..1b0c979 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,7 @@ Main limitations (non-exhaustive list): - Which version of GCC / clang did you compile the tests with? On which OS version? - If you intend to contribute a fix: - Please read [CONTRIBUTING](./CONTRIBUTING.md) first. You'd have to sign a CLA, which your employer may need to accept. - - Please test as many gated models as possible + - Please test as many gated models as possible (use `cmake -B build -DMINJA_TEST_GATED_MODELS=1 ...` and edit `MODEL_IDS` in [tests/CMakeLists.txt](./tests/CMakeLists.txt) appropriately if you don't have access to some models) - For bonus points, check the style of your edits with: diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index eb4354d..36bee3e 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -13,6 +13,6 @@ # set -euo pipefail -cmake -B build && \ +cmake -B build "$@" && \ cmake --build build -j && \ ctest --test-dir build -j --output-on-failure diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1277813..eaae929 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -50,18 +50,24 @@ set(MODEL_IDS "Qwen/Qwen2.5-Math-7B-Instruct" "teknium/OpenHermes-2.5-Mistral-7B" "TheBloke/FusionNet_34Bx2_MoE-AWQ" - - # Gated models: you will need to run `huggingface-cli login` (and be granted access) to download these - # "meta-llama/Llama-3.2-3B-Instruct" - # "meta-llama/Meta-Llama-3.1-8B-Instruct" - # "google/gemma-7b-it" - # "google/gemma-2-2b-it" - # "mistralai/Mistral-7B-Instruct-v0.2" - # "mistralai/Mixtral-8x7B-Instruct-v0.1" - # "mistralai/Mistral-Nemo-Instruct-2407", - # "CohereForAI/c4ai-command-r-plus" ) +# Gated models: you will need to run `huggingface-cli login` (and be granted access) to download these +if (MINJA_TEST_GATED_MODELS) + list(APPEND MODEL_IDS + "meta-llama/Llama-3.2-3B-Instruct" + "meta-llama/Meta-Llama-3.1-8B-Instruct" + "google/gemma-7b-it" + "google/gemma-2-2b-it" + "mistralai/Mistral-7B-Instruct-v0.2" + "mistralai/Mixtral-8x7B-Instruct-v0.1" + "mistralai/Mistral-Large-Instruct-2407" + "mistralai/Mistral-Large-Instruct-2411" + "mistralai/Mistral-Nemo-Instruct-2407" + "CohereForAI/c4ai-command-r-plus" + ) +endif() + # Create one test case for each {template, context} combination file(GLOB CONTEXT_FILES "${CMAKE_SOURCE_DIR}/tests/contexts/*.json") execute_process( @@ -81,13 +87,15 @@ foreach(test_case ${CHAT_TEMPLATE_TEST_CASES}) add_test(NAME ${test_name} COMMAND $ ${test_args}) endforeach() -if (FUZZTEST_FUZZING_MODE) - message(STATUS "Fuzzing mode enabled") - fuzztest_setup_fuzzing_flags() +if (MINJA_FUZZTEST_ENABLED) + if (MINJA_FUZZTEST_FUZZING_MODE) + message(STATUS "Fuzzing mode enabled") + fuzztest_setup_fuzzing_flags() + endif() + add_executable(test-fuzz test-fuzz.cpp) + set_target_properties(test-fuzz PROPERTIES CXX_STANDARD 17) + target_include_directories(test-fuzz PRIVATE ${fuzztest_BINARY_DIR}) + target_link_libraries(test-fuzz PRIVATE nlohmann_json::nlohmann_json) + link_fuzztest(test-fuzz) + gtest_discover_tests(test-fuzz) endif() -add_executable(test-fuzz test-fuzz.cpp) -set_target_properties(test-fuzz PROPERTIES CXX_STANDARD 17) -target_include_directories(test-fuzz PRIVATE ${fuzztest_BINARY_DIR}) -target_link_libraries(test-fuzz PRIVATE nlohmann_json::nlohmann_json) -link_fuzztest(test-fuzz) -gtest_discover_tests(test-fuzz)