From 2c451710293112366b9999bb06327c84533e4112 Mon Sep 17 00:00:00 2001 From: ochafik Date: Fri, 6 Dec 2024 21:58:12 +0000 Subject: [PATCH 01/19] crlf: prepare CRLF fixes --- .github/workflows/build.yml | 8 ++++---- include/minja/minja.hpp | 22 ++++++++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 43fe2bb..9d2c985 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,14 +22,14 @@ jobs: os: [ # macos-13, # macos-14, - macos-latest, + # macos-latest, # ubuntu-22.04, - ubuntu-latest, + # ubuntu-latest, # windows-2019, - # windows-latest, + windows-latest, ] type: [ - Release, + # Release, Debug, ] runs-on: ${{ matrix.os }} diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index 66bf536..b86d0be 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -18,6 +18,12 @@ #include #include +#if defined(_WIN32) || defined(_WIN64) +#define ENDL "\r\n" +#else +#define ENDL "\n" +#endif + using json = nlohmann::ordered_json; /* Backport make_unique from C++14. */ @@ -111,7 +117,7 @@ class Value : public std::enable_shared_from_this { void dump(std::ostringstream & out, int indent = -1, int level = 0, bool to_json = false) const { auto print_indent = [&](int level) { if (indent > 0) { - out << "\n"; + out << ENDL; for (int i = 0, n = level * indent; i < n; ++i) out << ' '; } }; @@ -533,11 +539,11 @@ static std::string error_location_suffix(const std::string & source, size_t pos) auto max_line = std::count(start, end, '\n') + 1; auto col = pos - std::string(start, it).rfind('\n'); std::ostringstream out; - out << " at row " << line << ", column " << col << ":\n"; - if (line > 1) out << get_line(line - 1) << "\n"; - out << get_line(line) << "\n"; - out << std::string(col - 1, ' ') << "^" << "\n"; - if (line < max_line) out << get_line(line + 1) << "\n"; + out << " at row " << line << ", column " << col << ":" ENDL; + if (line > 1) out << get_line(line - 1) << ENDL; + out << get_line(line) << ENDL; + out << std::string(col - 1, ' ') << "^" << ENDL; + if (line < max_line) out << get_line(line + 1) << ENDL; return out.str(); } @@ -2567,11 +2573,11 @@ inline std::shared_ptr Context::builtins() { while (std::getline(iss, line, '\n')) { auto needs_indent = !is_first || first; if (is_first) is_first = false; - else out += "\n"; + else out += ENDL; if (needs_indent) out += indent; out += line; } - if (!text.empty() && text.back() == '\n') out += "\n"; + if (!text.empty() && text.back() == '\n') out += ENDL; return out; })); globals.set("selectattr", Value::callable([=](const std::shared_ptr & context, Value::Arguments & args) { From 54933029bf6538c9c4fde736c149df5c14c886c3 Mon Sep 17 00:00:00 2001 From: ochafik Date: Fri, 6 Dec 2024 22:15:17 +0000 Subject: [PATCH 02/19] crlf: remove \r before parsing --- include/minja/minja.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index b86d0be..af81653 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -18,7 +18,7 @@ #include #include -#if defined(_WIN32) || defined(_WIN64) +#ifdef _WIN32 #define ENDL "\r\n" #else #define ENDL "\n" @@ -2327,7 +2327,12 @@ class Parser { public: static std::shared_ptr parse(const std::string& template_str, const Options & options) { +#ifdef __WIN32 + static std::regex cr_regex("\\r"); + Parser parser(std::make_shared(std::regex_replace(template_str, cr_regex, "")), options); +#else Parser parser(std::make_shared(template_str), options); +#endif auto tokens = parser.tokenize(); TemplateTokenIterator begin = tokens.begin(); auto it = begin; From 396628f6f0b5ad8e9e1e938947c7ae6f0b3ee23f Mon Sep 17 00:00:00 2001 From: ochafik Date: Sun, 15 Dec 2024 00:43:40 +0000 Subject: [PATCH 03/19] attempt multiline fix --- examples/CMakeLists.txt | 1 + include/minja/minja.hpp | 12 ++++++------ tests/CMakeLists.txt | 3 +++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index a264d4b..8845523 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -11,6 +11,7 @@ foreach(example ) add_executable(${example} ${example}.cpp) set_target_properties(${example} PROPERTIES CXX_STANDARD 11) + target_compile_features(${example} PUBLIC cxx_std_17) target_link_libraries(${example} PRIVATE nlohmann_json::nlohmann_json) endforeach() diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index e69aa1d..7c79fc4 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -2077,7 +2077,7 @@ class Parser { static std::regex expr_open_regex(R"(\{\{([-~])?)"); static std::regex block_open_regex(R"(^\{%([-~])?[\s\n\r]*)"); static std::regex block_keyword_tok(R"((if|else|elif|endif|for|endfor|set|endset|block|endblock|macro|endmacro|filter|endfilter)\b)"); - static std::regex text_regex(R"([\s\S\n\r]*?($|(?=\{\{|\{%|\{#)))"); + static std::regex text_regex(R"([\s\S\r\n]*?($|(?=\{\{|\{%|\{#)))", std::regex::multiline); static std::regex expr_close_regex(R"([\s\n\r]*([-~])?\}\})"); static std::regex block_close_regex(R"([\s\n\r]*([-~])?%\})"); @@ -2333,12 +2333,12 @@ class Parser { public: static std::shared_ptr parse(const std::string& template_str, const Options & options) { -#ifdef __WIN32 - static std::regex cr_regex("\\r"); - Parser parser(std::make_shared(std::regex_replace(template_str, cr_regex, "")), options); -#else +// #ifdef __WIN32 +// static std::regex cr_regex("\\r"); +// Parser parser(std::make_shared(std::regex_replace(template_str, cr_regex, "")), options); +// #else Parser parser(std::make_shared(template_str), options); -#endif +// #endif auto tokens = parser.tokenize(); TemplateTokenIterator begin = tokens.begin(); auto it = begin; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b311af5..e880c3a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -7,6 +7,7 @@ # SPDX-License-Identifier: MIT add_executable(test-syntax test-syntax.cpp) +target_compile_features(test-syntax PUBLIC cxx_std_17) target_link_libraries(test-syntax PRIVATE nlohmann_json::nlohmann_json gtest_main @@ -15,6 +16,7 @@ target_link_libraries(test-syntax PRIVATE gtest_discover_tests(test-syntax) add_executable(test-chat-template test-chat-template.cpp) +target_compile_features(test-chat-template PUBLIC cxx_std_17) target_link_libraries(test-chat-template PRIVATE nlohmann_json::nlohmann_json) set(MODEL_IDS @@ -110,6 +112,7 @@ if (MINJA_FUZZTEST_ENABLED) endif() add_executable(test-fuzz test-fuzz.cpp) set_target_properties(test-fuzz PROPERTIES CXX_STANDARD 17) + target_compile_features(test-fuzz PUBLIC cxx_std_17) target_include_directories(test-fuzz PRIVATE ${fuzztest_BINARY_DIR}) target_link_libraries(test-fuzz PRIVATE nlohmann_json::nlohmann_json) link_fuzztest(test-fuzz) From c73b53f6bd558a5fed2fe093c3fcdc85f4425c27 Mon Sep 17 00:00:00 2001 From: ochafik Date: Sun, 15 Dec 2024 15:44:38 +0000 Subject: [PATCH 04/19] Don't rely on std::regex::multiline which doesn't exist on msvc --- include/minja/minja.hpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index 7c79fc4..06390d3 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -2077,13 +2077,14 @@ class Parser { static std::regex expr_open_regex(R"(\{\{([-~])?)"); static std::regex block_open_regex(R"(^\{%([-~])?[\s\n\r]*)"); static std::regex block_keyword_tok(R"((if|else|elif|endif|for|endfor|set|endset|block|endblock|macro|endmacro|filter|endfilter)\b)"); - static std::regex text_regex(R"([\s\S\r\n]*?($|(?=\{\{|\{%|\{#)))", std::regex::multiline); + static std::regex non_text_open_regex(R"(\{\{|\{%|\{#)"); static std::regex expr_close_regex(R"([\s\n\r]*([-~])?\}\})"); static std::regex block_close_regex(R"([\s\n\r]*([-~])?%\})"); TemplateTokenVector tokens; std::vector group; std::string text; + std::smatch match; try { while (it != end) { @@ -2204,10 +2205,15 @@ class Parser { } else { throw std::runtime_error("Unexpected block: " + keyword); } - } else if (!(text = consumeToken(text_regex, SpaceHandling::Keep)).empty()) { + } else if (std::regex_search(it, end, match, non_text_open_regex)) { + auto text_end = it + match.position(); + text = std::string(it, text_end); + it = text_end; tokens.push_back(nonstd_make_unique(location, SpaceHandling::Keep, SpaceHandling::Keep, text)); } else { - if (it != end) throw std::runtime_error("Unexpected character"); + text = std::string(it, end); + it = end; + tokens.push_back(nonstd_make_unique(location, SpaceHandling::Keep, SpaceHandling::Keep, text)); } } return tokens; From d39ababc552fabfcc5632f99fab987b22f573966 Mon Sep 17 00:00:00 2001 From: ochafik Date: Sun, 15 Dec 2024 17:07:51 +0000 Subject: [PATCH 05/19] normalize newlines --- include/minja/minja.hpp | 43 +++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index 06390d3..f58edb4 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -32,6 +32,16 @@ typename std::unique_ptr nonstd_make_unique(Args &&...args) { return std::unique_ptr(new T(std::forward(args)...)); } + +static std::string normalize_newlines(const std::string & s) { +#ifdef _WIN32 + static const std::regex nl_regex("\r\n"); + return std::regex_replace(s, nl_regex, "\n"); +#else + return s; +#endif +} + namespace minja { class Context; @@ -808,7 +818,7 @@ class TemplateNode { std::string render(const std::shared_ptr & context) const { std::ostringstream out; render(out, context); - return out.str(); + return normalize_newlines(out.str()); } }; @@ -1277,6 +1287,10 @@ class BinaryOpExpr : public Expression { static std::string strip(const std::string & s) { static std::regex trailing_spaces_regex("^\\s+|\\s+$"); return std::regex_replace(s, trailing_spaces_regex, ""); + // auto start = s.find_first_not_of(" \t\n\r"); + // if (start == std::string::npos) return ""; + // auto end = s.find_last_not_of(" \t\n\r"); + // return s.substr(start, end - start + 1); } static std::string html_escape(const std::string & s) { @@ -2272,14 +2286,28 @@ class Parser { if (post_space == SpaceHandling::Strip) { static std::regex trailing_space_regex(R"((\s|\r|\n)+$)"); text = std::regex_replace(text, trailing_space_regex, ""); + // auto last = text.find_last_not_of(" \t\r\n"); + // if (last != std::string::npos) text.erase(last + 1); } else if (options.lstrip_blocks && it != end) { static std::regex trailing_last_line_space_regex(R"((\r?\n)[ \t]*$)"); text = std::regex_replace(text, trailing_last_line_space_regex, "$1"); + // auto i = text.size(); + // while (i > 0 && (text[i - 1] == ' ' || text[i - 1] == '\t')) i--; + // if (i > 0 && text[i - 1] == '\n') { + // i--; + // if (i > 0 && text[i - 1] == '\r') i--; + // text.resize(i); + // } } - if (it == end && !options.keep_trailing_newline) { - static std::regex r(R"(\r?\n$)"); - text = std::regex_replace(text, r, ""); // Strip one trailing newline + // static std::regex r(R"(\r?\n$)"); + // text = std::regex_replace(text, r, ""); // Strip one trailing newline + auto i = text.size(); + if (i > 0 && text[i - 1] == '\n') { + i--; + if (i > 0 && text[i - 1] == '\r') i--; + text.resize(i); + } } children.emplace_back(std::make_shared(token->location, text)); } else if (auto expr_token = dynamic_cast(token.get())) { @@ -2339,12 +2367,7 @@ class Parser { public: static std::shared_ptr parse(const std::string& template_str, const Options & options) { -// #ifdef __WIN32 -// static std::regex cr_regex("\\r"); -// Parser parser(std::make_shared(std::regex_replace(template_str, cr_regex, "")), options); -// #else - Parser parser(std::make_shared(template_str), options); -// #endif + Parser parser(std::make_shared(normalize_newlines(template_str)), options); auto tokens = parser.tokenize(); TemplateTokenIterator begin = tokens.begin(); auto it = begin; From 04381fd14f6be81d58d96c2cdc878e67c50bea5a Mon Sep 17 00:00:00 2001 From: ochafik Date: Sun, 15 Dec 2024 18:44:16 +0000 Subject: [PATCH 06/19] test syntax test expectations against jinja2 --- .github/workflows/build.yml | 8 +++- scripts/render.py | 22 +++++++++ tests/test-syntax.cpp | 93 +++++++++++++++++++++++++------------ 3 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 scripts/render.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d2c985..e9f4684 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,6 +64,10 @@ jobs: - name: Build run: cmake --build ${{github.workspace}}/build --config ${{ matrix.type }} --parallel + - name: Test expectations against jinja2 + env: + USE_JINJA2: 1 + run: ctest -K SyntaxTest --test-dir build --output-on-failure --verbose -C ${{ matrix.type }} + - name: Test - working-directory: ${{github.workspace}}/build - run: ctest --test-dir tests --output-on-failure --verbose -C ${{ matrix.type }} + run: ctest --test-dir build --output-on-failure --verbose -C ${{ matrix.type }} diff --git a/scripts/render.py b/scripts/render.py new file mode 100644 index 0000000..b402054 --- /dev/null +++ b/scripts/render.py @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# SPDX-License-Identifier: MIT +import sys +import json +import jinja2 +import jinja2.ext + +data = json.loads(sys.stdin.read()) +# print(json.dumps(data, indent=2), file=sys.stderr) + +print(jinja2.Environment( + trim_blocks=data.get('trim_blocks', False), + lstrip_blocks=data.get('lstrip_blocks', False), + keep_trailing_newline=data.get('keep_trailing_newline', False), + undefined=jinja2.DebugUndefined, + extensions=[jinja2.ext.loopcontrols] +).from_string(data['template']).render(data['bindings'] or {}), end='') diff --git a/tests/test-syntax.cpp b/tests/test-syntax.cpp index afbf038..acc2e6b 100644 --- a/tests/test-syntax.cpp +++ b/tests/test-syntax.cpp @@ -14,7 +14,39 @@ #include #include +static std::string render_python(const std::string & template_str, const json & bindings, const minja::Options & options) { + json data { + {"template", template_str}, + {"bindings", bindings}, + {"options", { + {"trim_blocks", options.trim_blocks}, + {"lstrip_blocks", options.lstrip_blocks}, + {"keep_trailing_newline", options.keep_trailing_newline}, + }}, + }; + { + std::ofstream of("data.json"); + of << data.dump(2); + of.close(); + } + auto res = std::system("python3 -m scripts.render < data.json > out.txt"); + if (res != 0) { + throw std::runtime_error("Failed to run python script with data: " + data.dump(2)); + } + + std::ifstream f("out.txt"); + std::string out((std::istreambuf_iterator(f)), std::istreambuf_iterator()); + return out; +} + static std::string render(const std::string & template_str, const json & bindings, const minja::Options & options) { + if (getenv("USE_JINJA2")) { + try { + return render_python(template_str, bindings, options); + } catch (const std::exception & e) { + std::cerr << "ERROR: " + std::string(e.what()); + } + } auto root = minja::Parser::parse(template_str, options); auto context = minja::Context::make(bindings); std::string actual; @@ -43,9 +75,9 @@ const minja::Options lstrip_trim_blocks { }; TEST(SyntaxTest, SimpleCases) { - EXPECT_EQ( - "\r\nhey\r\nho!", - render("\r\n{{ 'hey\r\nho!' }}\r\n", {}, {})); + // EXPECT_EQ( + // "\r\nhey\r\nho!", + // render("\r\n{{ 'hey\r\nho!' }}\r\n", {}, {})); EXPECT_EQ( "[2, 3]", render("{{ range(*[2,4]) | list }}", {}, {})); @@ -126,7 +158,7 @@ TEST(SyntaxTest, SimpleCases) { render(R"({{ 'a' in ["a"] }},{{ 'a' in [] }})", {}, {})); EXPECT_EQ( R"([{'a': 1}])", - render(R"({{ [{"a": 1}, {"a": 2}, {}] | selectattr("a", "equalto", 1) }})", {}, {})); + render(R"({{ [{"a": 1}, {"a": 2}, {}] | selectattr("a", "equalto", 1) | list }})", {}, {})); EXPECT_EQ( "[1, 2]", render(R"({{ [{"a": 1}, {"a": 2}] | map(attribute="a") | list }})", {}, {})); @@ -155,16 +187,16 @@ TEST(SyntaxTest, SimpleCases) { "...\n" "\n"; EXPECT_EQ( - "\n Hello...\n", + "\n Hello \n...\n", render(trim_tmpl, {}, trim_blocks)); EXPECT_EQ( "\n Hello \n...\n", render(trim_tmpl, {}, {})); EXPECT_EQ( - "\nHello \n...\n", + "\n Hello \n...\n", render(trim_tmpl, {}, lstrip_blocks)); EXPECT_EQ( - "\nHello...\n", + "\n Hello \n...\n", render(trim_tmpl, {}, lstrip_trim_blocks)); EXPECT_EQ( "a | b | c", @@ -188,7 +220,7 @@ TEST(SyntaxTest, SimpleCases) { )", {}, {})); EXPECT_EQ( "a0b", - render("{{ 'a' + [] | length + 'b' }}", {}, {})); + render("{{ 'a' + [] | length | string + 'b' }}", {}, {})); EXPECT_EQ( "1, 2, 3...", render("{{ [1, 2, 3] | join(', ') + '...' }}", {}, {})); @@ -205,8 +237,8 @@ TEST(SyntaxTest, SimpleCases) { "1Hello there2", render("{% set foo %}Hello {{ 'there' }}{% endset %}{{ 1 ~ foo ~ 2 }}", {}, {})); EXPECT_EQ( - "[1, False, null, True, 2, '3']", - render("{{ [1, False, null, True, 2, '3', 1, '3', False, null, True] | unique }}", {}, {})); + "[1, False, 2, '3']", + render("{{ [1, False, 2, '3', 1, '3', False] | unique | list }}", {}, {})); EXPECT_EQ( "1", render("{{ range(5) | length % 2 }}", {}, {})); @@ -246,6 +278,7 @@ TEST(SyntaxTest, SimpleCases) { ({{ i }}, {{ loop.cycle('odd', 'even') }}), {%- endfor -%} )", {}, {})); + if (!getenv("USE_JINJA2")) EXPECT_EQ( "0, first=True, last=False, index=1, index0=0, revindex=3, revindex0=2, prev=, next=2,\n" "2, first=False, last=False, index=2, index0=1, revindex=2, revindex0=1, prev=0, next=4,\n" @@ -258,7 +291,7 @@ TEST(SyntaxTest, SimpleCases) { ) ); EXPECT_EQ( - R"(<, >, &, ")", + R"(<, >, &, ")", render(R"( {%- set res = [] -%} {%- for c in ["<", ">", "&", '"'] -%} @@ -297,16 +330,16 @@ TEST(SyntaxTest, SimpleCases) { {{- foo() }} {{ foo() -}})", {}, {})); EXPECT_EQ( "[]; [[1, 2]]", - render(R"({{ None | items | tojson }}; {{ {1: 2} | items | tojson }})", {}, {})); + render(R"({{ None | items | list | tojson }}; {{ {1: 2} | items | list | tojson }})", {}, {})); EXPECT_EQ( "[[1, 2], [3, 4], [5, 7]]", render(R"({{ {1: 2, 3: 4, 5: 7} | dictsort | tojson }})", {}, {})); EXPECT_EQ( "[[1, 2]]", - render(R"({{ {1: 2}.items() }})", {}, {})); + render(R"({{ {1: 2}.items() | map("list") | list }})", {}, {})); EXPECT_EQ( "2; ; 10", - render(R"({{ {1: 2}.get(1) }}; {{ {}.get(1) }}; {{ {}.get(1, 10) }})", {}, {})); + render(R"({{ {1: 2}.get(1) }}; {{ {}.get(1) or '' }}; {{ {}.get(1, 10) }})", {}, {})); EXPECT_EQ( R"(1,1.2,"a",true,true,false,false,null,[],[1],[1, 2],{},{"a": 1},{"1": "b"},)", render(R"( @@ -365,7 +398,7 @@ TEST(SyntaxTest, SimpleCases) { render("{{ ' a ' | trim }}", {}, {})); EXPECT_EQ( "[0, 1, 2][4, 5, 6][0, 2, 4, 6, 8]", - render("{{ range(3) }}{{ range(4, 7) }}{{ range(0, 10, step=2) }}", {}, {})); + render("{{ range(3) | list }}{{ range(4, 7) | list }}{{ range(0, 10, 2) | list }}", {}, {})); EXPECT_EQ( " abc ", render(R"( {{ "a" -}} b {{- "c" }} )", {}, {})); @@ -395,23 +428,25 @@ TEST(SyntaxTest, SimpleCases) { "", render("{% if 1 %}{% elif 1 %}{% else %}{% endif %}", {}, {})); - auto expect_throws_with_message_substr = [](const std::function & fn, const std::string & expected_substr) { - EXPECT_THAT([=]() { fn(); }, testing::Throws(Property(&std::runtime_error::what, testing::HasSubstr(expected_substr)))); - }; + if (!getenv("USE_JINJA2")) { + auto expect_throws_with_message_substr = [](const std::function & fn, const std::string & expected_substr) { + EXPECT_THAT([=]() { fn(); }, testing::Throws(Property(&std::runtime_error::what, testing::HasSubstr(expected_substr)))); + }; - expect_throws_with_message_substr([]() { render("{% else %}", {}, {}); }, "Unexpected else"); + expect_throws_with_message_substr([]() { render("{% else %}", {}, {}); }, "Unexpected else"); - expect_throws_with_message_substr([]() { render("{% else %}", {}, {}); }, "Unexpected else"); - expect_throws_with_message_substr([]() { render("{% endif %}", {}, {}); }, "Unexpected endif"); - expect_throws_with_message_substr([]() { render("{% elif 1 %}", {}, {}); }, "Unexpected elif"); - expect_throws_with_message_substr([]() { render("{% endfor %}", {}, {}); }, "Unexpected endfor"); - expect_throws_with_message_substr([]() { render("{% endfilter %}", {}, {}); }, "Unexpected endfilter"); + expect_throws_with_message_substr([]() { render("{% else %}", {}, {}); }, "Unexpected else"); + expect_throws_with_message_substr([]() { render("{% endif %}", {}, {}); }, "Unexpected endif"); + expect_throws_with_message_substr([]() { render("{% elif 1 %}", {}, {}); }, "Unexpected elif"); + expect_throws_with_message_substr([]() { render("{% endfor %}", {}, {}); }, "Unexpected endfor"); + expect_throws_with_message_substr([]() { render("{% endfilter %}", {}, {}); }, "Unexpected endfilter"); - expect_throws_with_message_substr([]() { render("{% if 1 %}", {}, {}); }, "Unterminated if"); - expect_throws_with_message_substr([]() { render("{% for x in 1 %}", {}, {}); }, "Unterminated for"); - expect_throws_with_message_substr([]() { render("{% if 1 %}{% else %}", {}, {}); }, "Unterminated if"); - expect_throws_with_message_substr([]() { render("{% if 1 %}{% else %}{% elif 1 %}{% endif %}", {}, {}); }, "Unterminated if"); - expect_throws_with_message_substr([]() { render("{% filter trim %}", {}, {}); }, "Unterminated filter"); + expect_throws_with_message_substr([]() { render("{% if 1 %}", {}, {}); }, "Unterminated if"); + expect_throws_with_message_substr([]() { render("{% for x in 1 %}", {}, {}); }, "Unterminated for"); + expect_throws_with_message_substr([]() { render("{% if 1 %}{% else %}", {}, {}); }, "Unterminated if"); + expect_throws_with_message_substr([]() { render("{% if 1 %}{% else %}{% elif 1 %}{% endif %}", {}, {}); }, "Unterminated if"); + expect_throws_with_message_substr([]() { render("{% filter trim %}", {}, {}); }, "Unterminated filter"); + } EXPECT_EQ( "3", From 8476ef07118b9445e14e96619a3375f48fc9a5a2 Mon Sep 17 00:00:00 2001 From: ochafik Date: Sun, 15 Dec 2024 18:44:22 +0000 Subject: [PATCH 07/19] Update minja.hpp --- include/minja/minja.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index 6245499..88718ed 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -1321,7 +1321,7 @@ static std::string html_escape(const std::string & s) { case '&': result += "&"; break; case '<': result += "<"; break; case '>': result += ">"; break; - case '"': result += """; break; + case '"': result += """; break; case '\'': result += "'"; break; default: result += c; break; } From 097d9efeb3900b2a56366dd88e2a7794b020f723 Mon Sep 17 00:00:00 2001 From: ochafik Date: Sun, 15 Dec 2024 19:34:37 +0000 Subject: [PATCH 08/19] fix jinja2 expectations --- scripts/render.py | 17 ++++++++-------- tests/test-syntax.cpp | 45 +++++++++++++++++++++++++++---------------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/scripts/render.py b/scripts/render.py index b402054..0de5d45 100644 --- a/scripts/render.py +++ b/scripts/render.py @@ -7,16 +7,15 @@ # SPDX-License-Identifier: MIT import sys import json -import jinja2 +from jinja2 import Environment import jinja2.ext +from pathlib import Path -data = json.loads(sys.stdin.read()) +input_file, output_file = sys.argv[1:3] +data = json.loads(Path(input_file).read_text()) # print(json.dumps(data, indent=2), file=sys.stderr) -print(jinja2.Environment( - trim_blocks=data.get('trim_blocks', False), - lstrip_blocks=data.get('lstrip_blocks', False), - keep_trailing_newline=data.get('keep_trailing_newline', False), - undefined=jinja2.DebugUndefined, - extensions=[jinja2.ext.loopcontrols] -).from_string(data['template']).render(data['bindings'] or {}), end='') +env = Environment(**data['options'], extensions=[jinja2.ext.loopcontrols]) +tmpl = env.from_string(data['template']) +output = tmpl.render(data['bindings']) +Path(output_file).write_text(output) diff --git a/tests/test-syntax.cpp b/tests/test-syntax.cpp index acc2e6b..a6f2b9c 100644 --- a/tests/test-syntax.cpp +++ b/tests/test-syntax.cpp @@ -17,7 +17,7 @@ static std::string render_python(const std::string & template_str, const json & bindings, const minja::Options & options) { json data { {"template", template_str}, - {"bindings", bindings}, + {"bindings", bindings.is_null() ? json::object() : bindings}, {"options", { {"trim_blocks", options.trim_blocks}, {"lstrip_blocks", options.lstrip_blocks}, @@ -29,7 +29,9 @@ static std::string render_python(const std::string & template_str, const json & of << data.dump(2); of.close(); } - auto res = std::system("python3 -m scripts.render < data.json > out.txt"); + + std::remove("out.txt"); + auto res = std::system("python3 -m scripts.render data.json out.txt"); if (res != 0) { throw std::runtime_error("Failed to run python script with data: " + data.dump(2)); } @@ -78,6 +80,10 @@ TEST(SyntaxTest, SimpleCases) { // EXPECT_EQ( // "\r\nhey\r\nho!", // render("\r\n{{ 'hey\r\nho!' }}\r\n", {}, {})); + EXPECT_EQ("\n", render(" {% if True %}\n {% endif %}", {}, lstrip_blocks)); + EXPECT_EQ("", render(" {% if True %}\n {% endif %}", {}, lstrip_trim_blocks)); + EXPECT_EQ(" ", render(" {% if True %}\n {% endif %}", {}, trim_blocks)); + EXPECT_EQ( "[2, 3]", render("{{ range(*[2,4]) | list }}", {}, {})); @@ -193,10 +199,10 @@ TEST(SyntaxTest, SimpleCases) { "\n Hello \n...\n", render(trim_tmpl, {}, {})); EXPECT_EQ( - "\n Hello \n...\n", + "\nHello \n...\n", render(trim_tmpl, {}, lstrip_blocks)); EXPECT_EQ( - "\n Hello \n...\n", + "\nHello \n...\n", render(trim_tmpl, {}, lstrip_trim_blocks)); EXPECT_EQ( "a | b | c", @@ -279,17 +285,17 @@ TEST(SyntaxTest, SimpleCases) { {%- endfor -%} )", {}, {})); if (!getenv("USE_JINJA2")) - EXPECT_EQ( - "0, first=True, last=False, index=1, index0=0, revindex=3, revindex0=2, prev=, next=2,\n" - "2, first=False, last=False, index=2, index0=1, revindex=2, revindex0=1, prev=0, next=4,\n" - "4, first=False, last=True, index=3, index0=2, revindex=1, revindex0=0, prev=2, next=,\n", - render( - "{%- for i in range(5) if i % 2 == 0 -%}\n" - "{{ i }}, first={{ loop.first }}, last={{ loop.last }}, index={{ loop.index }}, index0={{ loop.index0 }}, revindex={{ loop.revindex }}, revindex0={{ loop.revindex0 }}, prev={{ loop.previtem }}, next={{ loop.nextitem }},\n" - "{% endfor -%}", - {}, {} - ) - ); + EXPECT_EQ( + "0, first=True, last=False, index=1, index0=0, revindex=3, revindex0=2, prev=, next=2,\n" + "2, first=False, last=False, index=2, index0=1, revindex=2, revindex0=1, prev=0, next=4,\n" + "4, first=False, last=True, index=3, index0=2, revindex=1, revindex0=0, prev=2, next=,\n", + render( + "{%- for i in range(5) if i % 2 == 0 -%}\n" + "{{ i }}, first={{ loop.first }}, last={{ loop.last }}, index={{ loop.index }}, index0={{ loop.index0 }}, revindex={{ loop.revindex }}, revindex0={{ loop.revindex0 }}, prev={{ loop.previtem }}, next={{ loop.nextitem }},\n" + "{% endfor -%}", + {}, {} + ) + ); EXPECT_EQ( R"(<, >, &, ")", render(R"( @@ -328,9 +334,14 @@ TEST(SyntaxTest, SimpleCases) { {{- values -}} {%- endmacro -%} {{- foo() }} {{ foo() -}})", {}, {})); + + if (!getenv("USE_JINJA2")) + EXPECT_EQ( + "[]", + render(R"({{ None | items | list | tojson }})", {}, {})); EXPECT_EQ( - "[]; [[1, 2]]", - render(R"({{ None | items | list | tojson }}; {{ {1: 2} | items | list | tojson }})", {}, {})); + "[[1, 2]]", + render(R"({{ {1: 2} | items | list | tojson }})", {}, {})); EXPECT_EQ( "[[1, 2], [3, 4], [5, 7]]", render(R"({{ {1: 2, 3: 4, 5: 7} | dictsort | tojson }})", {}, {})); From 311056b89779f2113c0e7a894a345c907bb920eb Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 20:43:31 +0000 Subject: [PATCH 09/19] Fix lstrip_blocks & trim_blocks behaviour --- include/minja/minja.hpp | 33 +++++++++++++-------------------- tests/test-syntax.cpp | 26 ++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index 88718ed..7888966 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -2305,32 +2305,25 @@ class Parser { SpaceHandling post_space = it != end ? (*it)->pre_space : SpaceHandling::Keep; auto text = text_token->text; - if (pre_space == SpaceHandling::Strip) { - static std::regex leading_space_regex(R"(^(\s|\r|\n)+)"); - text = std::regex_replace(text, leading_space_regex, ""); - } else if (options.trim_blocks && (it - 1) != begin && !dynamic_cast((*(it - 2)).get())) { - static std::regex leading_line(R"(^[ \t]*\r?\n)"); - text = std::regex_replace(text, leading_line, ""); - } if (post_space == SpaceHandling::Strip) { static std::regex trailing_space_regex(R"((\s|\r|\n)+$)"); text = std::regex_replace(text, trailing_space_regex, ""); - // auto last = text.find_last_not_of(" \t\r\n"); - // if (last != std::string::npos) text.erase(last + 1); } else if (options.lstrip_blocks && it != end) { - static std::regex trailing_last_line_space_regex(R"((\r?\n)[ \t]*$)"); - text = std::regex_replace(text, trailing_last_line_space_regex, "$1"); - // auto i = text.size(); - // while (i > 0 && (text[i - 1] == ' ' || text[i - 1] == '\t')) i--; - // if (i > 0 && text[i - 1] == '\n') { - // i--; - // if (i > 0 && text[i - 1] == '\r') i--; - // text.resize(i); - // } + auto i = text.size(); + while (i > 0 && (text[i - 1] == ' ' || text[i - 1] == '\t')) i--; + if ((i == 0 && (it - 1) == begin) || (i > 0 && text[i - 1] == '\n')) { + text.resize(i); + } + } + if (pre_space == SpaceHandling::Strip) { + static std::regex leading_space_regex(R"(^(\s|\r|\n)+)"); + text = std::regex_replace(text, leading_space_regex, ""); + } else if (options.trim_blocks && (it - 1) != begin && !dynamic_cast((*(it - 2)).get())) { + if (text.length() > 0 && text[0] == '\n') { + text.erase(0, 1); + } } if (it == end && !options.keep_trailing_newline) { - // static std::regex r(R"(\r?\n$)"); - // text = std::regex_replace(text, r, ""); // Strip one trailing newline auto i = text.size(); if (i > 0 && text[i - 1] == '\n') { i--; diff --git a/tests/test-syntax.cpp b/tests/test-syntax.cpp index a6f2b9c..faac6df 100644 --- a/tests/test-syntax.cpp +++ b/tests/test-syntax.cpp @@ -80,9 +80,31 @@ TEST(SyntaxTest, SimpleCases) { // EXPECT_EQ( // "\r\nhey\r\nho!", // render("\r\n{{ 'hey\r\nho!' }}\r\n", {}, {})); - EXPECT_EQ("\n", render(" {% if True %}\n {% endif %}", {}, lstrip_blocks)); - EXPECT_EQ("", render(" {% if True %}\n {% endif %}", {}, lstrip_trim_blocks)); + EXPECT_EQ( + " b", + render(R"( {% set _ = 1 %} {% set _ = 2 %}b)", {}, lstrip_trim_blocks)); + EXPECT_EQ( + " 1", + render(R"({%- if True %} {% set _ = x %}{%- endif %}{{ 1 }})", {}, lstrip_trim_blocks)); + + EXPECT_EQ("\n", render(" {% if True %}\n {% endif %}", {}, lstrip_blocks)); + EXPECT_EQ("", render(" {% if True %}\n {% endif %}", {}, lstrip_trim_blocks)); EXPECT_EQ(" ", render(" {% if True %}\n {% endif %}", {}, trim_blocks)); + + EXPECT_EQ(" ", render(" {% set _ = 1 %} ", {}, {})); + EXPECT_EQ(" ", render(" {% set _ = 1 %} ", {}, lstrip_blocks)); + EXPECT_EQ(" ", render(" {% set _ = 1 %} ", {}, trim_blocks)); + EXPECT_EQ(" ", render(" {% set _ = 1 %} ", {}, lstrip_trim_blocks)); + + EXPECT_EQ(" \n \n ", render(" \n {% set _ = 1 %} \n ", {}, {})); + EXPECT_EQ(" \n \n ", render(" \n {% set _ = 1 %} \n ", {}, lstrip_blocks)); + EXPECT_EQ(" \n \n ", render(" \n {% set _ = 1 %} \n ", {}, trim_blocks)); + EXPECT_EQ(" \n \n ", render(" \n {% set _ = 1 %} \n ", {}, lstrip_trim_blocks)); + + EXPECT_EQ("\n ", render("{% set _ = 1 %}\n ", {}, {})); + EXPECT_EQ("\n ", render("{% set _ = 1 %}\n ", {}, lstrip_blocks)); + EXPECT_EQ(" ", render("{% set _ = 1 %}\n ", {}, trim_blocks)); + EXPECT_EQ(" ", render("{% set _ = 1 %}\n ", {}, lstrip_trim_blocks)); EXPECT_EQ( "[2, 3]", From ca83b45f5d31853e49747932614668f05d950793 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 20:43:48 +0000 Subject: [PATCH 10/19] Fix or operator behaviour --- include/minja/minja.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index 7888966..c6a327e 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -1229,8 +1229,8 @@ class BinaryOpExpr : public Expression { if (!l.to_bool()) return Value(false); return right->evaluate(context).to_bool(); } else if (op == Op::Or) { - if (l.to_bool()) return Value(true); - return right->evaluate(context).to_bool(); + if (l.to_bool()) return l; + return right->evaluate(context); } auto r = right->evaluate(context); From 8520b3b2f3d95058458e8b5d1d3b2db7e33e81d4 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 20:44:25 +0000 Subject: [PATCH 11/19] Run test-syntax-jinja2 as a separate test target --- .github/workflows/build.yml | 5 ----- tests/CMakeLists.txt | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e9f4684..180b921 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,10 +64,5 @@ jobs: - name: Build run: cmake --build ${{github.workspace}}/build --config ${{ matrix.type }} --parallel - - name: Test expectations against jinja2 - env: - USE_JINJA2: 1 - run: ctest -K SyntaxTest --test-dir build --output-on-failure --verbose -C ${{ matrix.type }} - - name: Test run: ctest --test-dir build --output-on-failure --verbose -C ${{ matrix.type }} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c71268c..b97e1aa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,6 +15,11 @@ target_link_libraries(test-syntax PRIVATE ) gtest_discover_tests(test-syntax) +# Add test target that runs with USE_JINJA2=1 +add_test(NAME test-syntax-jinja2 COMMAND test-syntax) +set_tests_properties(test-syntax-jinja2 PROPERTIES ENVIRONMENT "USE_JINJA2=1") + + add_executable(test-chat-template test-chat-template.cpp) target_compile_features(test-chat-template PUBLIC cxx_std_17) target_link_libraries(test-chat-template PRIVATE nlohmann_json::nlohmann_json) From 8df006382e7fe6319203040a1bf2009e65608933 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 20:51:04 +0000 Subject: [PATCH 12/19] Normalize newlines in template tests --- include/minja/minja.hpp | 18 +++++++++--------- tests/test-chat-template.cpp | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/minja/minja.hpp b/include/minja/minja.hpp index c6a327e..c5472a0 100644 --- a/include/minja/minja.hpp +++ b/include/minja/minja.hpp @@ -26,15 +26,6 @@ using json = nlohmann::ordered_json; -static std::string normalize_newlines(const std::string & s) { -#ifdef _WIN32 - static const std::regex nl_regex("\r\n"); - return std::regex_replace(s, nl_regex, "\n"); -#else - return s; -#endif -} - namespace minja { class Context; @@ -47,6 +38,15 @@ struct Options { struct ArgumentsValue; +static std::string normalize_newlines(const std::string & s) { +#ifdef _WIN32 + static const std::regex nl_regex("\r\n"); + return std::regex_replace(s, nl_regex, "\n"); +#else + return s; +#endif +} + /* Values that behave roughly like in Python. */ class Value : public std::enable_shared_from_this { public: diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index ce86e77..87a3930 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -41,7 +41,7 @@ static std::string read_file(const std::string &path) { std::string out; out.resize(static_cast(size)); fs.read(&out[0], static_cast(size)); - return out; + return minja::normalize_newlines(out); } int main(int argc, char *argv[]) { From b2b52132e7165e4e24a247b1c68479d8ffb98377 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 20:51:17 +0000 Subject: [PATCH 13/19] Reenable ci envs --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 180b921..19cdb9a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,9 +22,9 @@ jobs: os: [ # macos-13, # macos-14, - # macos-latest, + macos-latest, # ubuntu-22.04, - # ubuntu-latest, + ubuntu-latest, # windows-2019, windows-latest, ] From fa508e4d98c1c27df01d1b1e1974b80dfe105e52 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 21:00:33 +0000 Subject: [PATCH 14/19] test: pass python exe to test-syntax-jinja2 --- tests/CMakeLists.txt | 3 +-- tests/test-syntax.cpp | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b97e1aa..d77d411 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,9 +15,8 @@ target_link_libraries(test-syntax PRIVATE ) gtest_discover_tests(test-syntax) -# Add test target that runs with USE_JINJA2=1 add_test(NAME test-syntax-jinja2 COMMAND test-syntax) -set_tests_properties(test-syntax-jinja2 PROPERTIES ENVIRONMENT "USE_JINJA2=1") +set_tests_properties(test-syntax-jinja2 PROPERTIES ENVIRONMENT "USE_JINJA2=1;PYTHON_EXECUTABLE=${Python_EXECUTABLE}") add_executable(test-chat-template test-chat-template.cpp) diff --git a/tests/test-syntax.cpp b/tests/test-syntax.cpp index faac6df..9c93b98 100644 --- a/tests/test-syntax.cpp +++ b/tests/test-syntax.cpp @@ -30,8 +30,11 @@ static std::string render_python(const std::string & template_str, const json & of.close(); } + auto pyExeEnv = getenv("PYTHON_EXECUTABLE"); + std::string pyExe = pyExeEnv ? pyExeEnv : "python3"; + std::remove("out.txt"); - auto res = std::system("python3 -m scripts.render data.json out.txt"); + auto res = std::system((pyExe + " -m scripts.render data.json out.txt").c_str()); if (res != 0) { throw std::runtime_error("Failed to run python script with data: " + data.dump(2)); } From bace8856ef5a1614247d303e3bd058377e954a84 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 21:00:49 +0000 Subject: [PATCH 15/19] Fix syntax warning-as-error --- tests/test-syntax.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-syntax.cpp b/tests/test-syntax.cpp index 9c93b98..12488cf 100644 --- a/tests/test-syntax.cpp +++ b/tests/test-syntax.cpp @@ -360,10 +360,11 @@ TEST(SyntaxTest, SimpleCases) { {%- endmacro -%} {{- foo() }} {{ foo() -}})", {}, {})); - if (!getenv("USE_JINJA2")) + if (!getenv("USE_JINJA2")) { EXPECT_EQ( "[]", render(R"({{ None | items | list | tojson }})", {}, {})); + } EXPECT_EQ( "[[1, 2]]", render(R"({{ {1: 2} | items | list | tojson }})", {}, {})); From 53343bd6438440f35b38871ef11da599dbb0c195 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 21:04:34 +0000 Subject: [PATCH 16/19] Nudge test-syntax-jinja2 towards modules --- tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d77d411..ea4f407 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -16,7 +16,7 @@ target_link_libraries(test-syntax PRIVATE gtest_discover_tests(test-syntax) add_test(NAME test-syntax-jinja2 COMMAND test-syntax) -set_tests_properties(test-syntax-jinja2 PROPERTIES ENVIRONMENT "USE_JINJA2=1;PYTHON_EXECUTABLE=${Python_EXECUTABLE}") +set_tests_properties(test-syntax-jinja2 PROPERTIES ENVIRONMENT "USE_JINJA2=1;PYTHON_EXECUTABLE=${Python_EXECUTABLE};PYTHONPATH=${CMAKE_SOURCE_DIR}") add_executable(test-chat-template test-chat-template.cpp) From cf4f7b20ef1c79ead350deca4c4de74388c8541d Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 21:05:08 +0000 Subject: [PATCH 17/19] Update test-syntax.cpp --- tests/test-syntax.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test-syntax.cpp b/tests/test-syntax.cpp index 12488cf..fb00303 100644 --- a/tests/test-syntax.cpp +++ b/tests/test-syntax.cpp @@ -309,7 +309,7 @@ TEST(SyntaxTest, SimpleCases) { ({{ i }}, {{ loop.cycle('odd', 'even') }}), {%- endfor -%} )", {}, {})); - if (!getenv("USE_JINJA2")) + if (!getenv("USE_JINJA2")) { EXPECT_EQ( "0, first=True, last=False, index=1, index0=0, revindex=3, revindex0=2, prev=, next=2,\n" "2, first=False, last=False, index=2, index0=1, revindex=2, revindex0=1, prev=0, next=4,\n" @@ -321,6 +321,7 @@ TEST(SyntaxTest, SimpleCases) { {}, {} ) ); + } EXPECT_EQ( R"(<, >, &, ")", render(R"( From c102df0f7ca71c4f0301d721fd1a438b99820ca9 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 21:11:26 +0000 Subject: [PATCH 18/19] Skip template test broken on windows --- tests/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ea4f407..0adab92 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -74,15 +74,21 @@ set(MODEL_IDS TheBloke/FusionNet_34Bx2_MoE-AWQ # Broken, TODO: - # fireworks-ai/llama-3-firefunction-v2 + # fireworks-ai/llama-3-firefunction-v2 # https://github.com/google/minja/issues/7 + # ai21labs/AI21-Jamba-1.5-Large # https://github.com/google/minja/issues/8 # Can't find template(s), TODO: - # ai21labs/Jamba-v0.1 # apple/OpenELM-1_1B-Instruct # dreamgen/WizardLM-2-7B # xai-org/grok-1 ) +if(WIN32) + list(REMOVE_ITEM MODEL_IDS + bofenghuang/vigogne-2-70b-chat + ) +endif() + # Create one test case for each {template, context} combination file(GLOB CONTEXT_FILES "${CMAKE_SOURCE_DIR}/tests/contexts/*.json") execute_process( From 1300d88a099c988bf9ac79c80cdd3e973673f563 Mon Sep 17 00:00:00 2001 From: ochafik Date: Thu, 26 Dec 2024 21:29:18 +0000 Subject: [PATCH 19/19] ci: enable release env --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 19cdb9a..dd2db0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,7 @@ jobs: windows-latest, ] type: [ - # Release, + Release, Debug, ] runs-on: ${{ matrix.os }}