Add ReasoningFormat detection and automatic polyfills

.github/workflows/build.yml

@@ -3,7 +3,7 @@ name: CI
 on:
   push:
     branches:
-      - master
+      - main
     paths: ['.github/workflows/build.yml', '**/CMakeLists.txt', '**/*.hpp', '**/*.cpp']
   pull_request:
     types: [opened, synchronize, reopened]
@@ -44,8 +44,29 @@ jobs:
           Release,
           Debug,
         ]
+        sanitizer: [
+          none,
+          address,
+          thread,
+          undefined,
+        ]
+        exclude:
+          # Sanitizers not supported on Clang targeting MSVC (llvm-arm64)
+          - setup: { build: 'llvm-arm64' }
+            sanitizer: address
+          - setup: { build: 'llvm-arm64' }
+            sanitizer: thread
+          - setup: { build: 'llvm-arm64' }
+            sanitizer: undefined
+          # Sanitizers not supported on MSVC ARM64
+          - setup: { build: 'msvc-arm64' }
+            sanitizer: address
+          - setup: { build: 'msvc-arm64' }
+            sanitizer: thread
+          - setup: { build: 'msvc-arm64' }
+            sanitizer: undefined
     runs-on: ${{ matrix.setup.os }}
-    name: ${{ matrix.setup.os }}-${{ matrix.setup.build }}-${{ matrix.type }}
+    name: ${{ matrix.setup.os }}-${{ matrix.setup.build }}-${{ matrix.type }}-sanitizer-${{ matrix.sanitizer }}
     timeout-minutes: 30
 
     steps:
@@ -58,7 +79,7 @@ jobs:
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2.11
         with:
-          key: ${{ matrix.setup.os }}-${{ matrix.setup.build }}-${{ matrix.type }}
+          key: ${{ matrix.setup.os }}-${{ matrix.setup.build }}-${{ matrix.type }}-sanitizer-${{ matrix.sanitizer }}
 
       - name: Set up CMake
         uses: lukka/get-cmake@latest
@@ -75,11 +96,14 @@ jobs:
       - name: Configure CMake
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        run: cmake -B ${{github.workspace}}/build ${{ matrix.setup.defines }} -DCMAKE_BUILD_TYPE=${{ matrix.type }}
+          PYTHONIOENCODING: utf-8
+        run: cmake -B ${{github.workspace}}/build ${{ matrix.setup.defines }} -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DMINJA_SANITIZER=${{ matrix.sanitizer }}
 
       - name: Build
         run: cmake --build ${{github.workspace}}/build --config ${{ matrix.type }} --parallel
 
       - name: Test
         if: ${{ matrix.setup.test }}
+        env:
+          PYTHONIOENCODING: utf-8
         run: ctest --test-dir build --output-on-failure --verbose -C ${{ matrix.type }}

CMakeLists.txt

@@ -15,9 +15,9 @@ add_library(minja INTERFACE)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-# Test if clang-tidy is available
+# Test if clang-tidy is available (disabled for address sanitizer due to GCC false positives)
 find_program(CLANG_TIDY_EXE NAMES "clang-tidy")
-if (CLANG_TIDY_EXE)
+if (CLANG_TIDY_EXE AND NOT MINJA_SANITIZER STREQUAL "address")
     message(STATUS "clang-tidy found: ${CLANG_TIDY_EXE}")
     set(CMAKE_CXX_CLANG_TIDY
         clang-tidy;
@@ -27,6 +27,8 @@ if (CLANG_TIDY_EXE)
         -checks=-*,clang-analyzer-*,clang-diagnostic-*,cppcoreguideline-*,bugprone-*,-bugprone-suspicious-include,-bugprone-assignment-in-if-condition,-bugprone-narrowing-conversions,-bugprone-easily-swappable-parameters,-bugprone-inc-dec-in-conditions,-bugprone-exception-escape,-clang-analyzer-cplusplus.StringChecker;
         -warnings-as-errors=*;
     )
+elseif(MINJA_SANITIZER STREQUAL "address")
+    message(STATUS "clang-tidy disabled for address sanitizer builds")
 else()
     message(STATUS "clang-tidy not found")
 endif()
@@ -43,13 +45,27 @@ option(MINJA_EXAMPLE_ENABLED        "minja: Build with example"
 option(MINJA_FUZZTEST_ENABLED       "minja: fuzztests enabled"                              MINJA_FUZZTEST_ENABLED_DEFAULT)
 option(MINJA_FUZZTEST_FUZZING_MODE  "minja: run fuzztests (if enabled) in fuzzing mode"     OFF)
 option(MINJA_USE_VENV               "minja: use Python venv for build"                      MINJA_USE_VENV_DEFAULT)
+set(MINJA_SANITIZERS thread address undefined none)
+set(MINJA_SANITIZER none CACHE STRING "minja: sanitizer to use")
+set_property(CACHE MINJA_SANITIZER PROPERTY STRINGS ${MINJA_SANITIZERS})
+
+if (NOT MSVC AND NOT CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC" AND NOT MINJA_SANITIZER STREQUAL "none")
+    message(STATUS "Using -fsanitize=${MINJA_SANITIZER}")
+    add_compile_options("-fsanitize=${MINJA_SANITIZER}")
+    link_libraries     ("-fsanitize=${MINJA_SANITIZER}")
+endif()
 
 set(CMAKE_CXX_STANDARD 17)
 
 set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL")
 set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
 if (NOT MSVC)
     add_compile_options(-Wall -Wextra -pedantic -Werror)
+    # GCC 13+ has false-positive maybe-uninitialized warnings with address sanitizer
+    # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105562
+    if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND MINJA_SANITIZER STREQUAL "address")
+        add_compile_options(-Wno-maybe-uninitialized)
+    endif()
 endif()
 
 include(FetchContent)
@@ -108,10 +124,13 @@ if(MINJA_TEST_ENABLED)
     message(STATUS "Python executable: ${Python_EXECUTABLE}")
 endif()
 
-find_program(CPPCHECK cppcheck)
-if(CPPCHECK)
-  set(CMAKE_CXX_CPPCHECK "${CPPCHECK}" -i ${json_SOURCE_DIR}/include/nlohmann/json.hpp)
-  message(STATUS "cppcheck found: ${CPPCHECK}")
+# cppcheck has issues on Windows (missing std.cfg), so we only enable it on non-Windows
+if(NOT WIN32)
+  find_program(CPPCHECK cppcheck)
+  if(CPPCHECK)
+    set(CMAKE_CXX_CPPCHECK "${CPPCHECK}" -i ${json_SOURCE_DIR}/include/nlohmann/json.hpp)
+    message(STATUS "cppcheck found: ${CPPCHECK}")
+  endif()
 endif()
 
 include(GNUInstallDirs)
@@ -135,6 +154,8 @@ if(MINJA_EXAMPLE_ENABLED)
     add_subdirectory(examples)
 endif()
 
+add_subdirectory(tools)
+
 if(MINJA_TEST_ENABLED)
     enable_testing()
     include(GoogleTest)

README.md

@@ -1,6 +1,6 @@
 # minja.hpp - A minimalistic C++ Jinja templating engine for LLM chat templates
 
-_**This is not an official Google product**_
+_**Used to be at https://github.com/google/minja, but I've left Google and I'll only maintain my fork from now on**_
 
 Minja is a minimalistic reimplementation of the [Jinja](https://github.com/pallets/jinja/) templating engine to integrate in/with C++ LLM projects (it's used in [llama.cpp](https://github.com/ggerganov/llama.cpp/pull/11016), [Jan](https://jan.ai/) (through [cortex.cpp](https://github.com/menloresearch/cortex.cpp/pull/1814)), [GPT4All](https://github.com/nomic-ai/gpt4all/pull/3433) and [Docker Model Runner](https://github.com/docker/model-runner)).
 
@@ -212,6 +212,26 @@ Main limitations (non-exhaustive list):
         ./scripts/fuzzing_tests.sh
         ```
 
+- Sanitizer tests:
+
+    ```bash
+    for sanitizer in ADDRESS THREAD UNDEFINED ; do
+        docker run --rm \
+            -v "$PWD":/src:ro \
+            -v "$PWD/build-sanitizer-${sanitizer}":/src/build \
+            -w /src \
+            "$(echo "
+                FROM ghcr.io/astral-sh/uv:debian-slim
+                RUN apt-get update && apt-get install -y build-essential libcurl4-openssl-dev cmake clang-tidy
+            " | docker build . -q -f - )" \
+            bash -c "
+                cmake -B build -DCMAKE_BUILD_TYPE=Debug -DMINJA_SANITIZER=${sanitizer} && \
+                cmake --build build -j --config Debug && \
+                ctest --test-dir build -j -C Debug --output-on-failure
+            "
+    done
+    ```
+
 - If your model's template doesn't run fine, please consider the following before [opening a bug](https://github.com/googlestaging/minja/issues/new):
 
     - Is the template using any unsupported filter / test / method / global function, and which one(s)?