Skip to content

build: loudly skip tests we fail to get templates of (e.g. gated) #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ochafik merged 3 commits into from
Dec 6, 2024
Merged

build: loudly skip tests we fail to get templates of (e.g. gated) #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0fd12b8
build: loudly skip tests we fail to get templates of (drop test gated…
Dec 6, 2024
3af6f85
models: test more models, loosely tag gated models as a comment
Dec 6, 2024
89c4847
models: restructure list of models (can't find templates of a few pop…
Dec 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ project(minja VERSION 1.0.0 LANGUAGES CXX)

set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

option(MINJA_TEST_GATED_MODELS "minja: test gated models" OFF)
if (MSVC)
set(MINJA_FUZZTEST_ENABLED_DEFAULT OFF)
else()
Expand Down
6 changes: 6 additions & 0 deletions scripts/fetch_templates_and_goldens.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ def handle_chat_template(output_folder, model_id, variant, template_src, context
with open(template_file, 'w') as f:
f.write(template_src)

if not context_files:
print(f"{template_file} n/a {template_file}")
return

env = jinja2.Environment(
trim_blocks=True,
lstrip_blocks=True,
Expand Down Expand Up @@ -147,6 +151,7 @@ def main():
except json.JSONDecodeError:
config = json.loads(re.sub(r'\}([\n\s]*\}[\n\s]*\],[\n\s]*"clean_up_tokenization_spaces")', r'\1', config_str))

assert 'chat_template' in config, 'No "chat_template" entry in tokenizer_config.json!'
chat_template = config['chat_template']
if isinstance(chat_template, str):
handle_chat_template(output_folder, model_id, None, chat_template, context_files)
Expand All @@ -155,6 +160,7 @@ def main():
handle_chat_template(output_folder, model_id, ct['name'], ct['template'], context_files)
except Exception as e:
logger.error(f"Error processing model {model_id}: {e}")
handle_chat_template(output_folder, model_id, None, str(e), [])


if __name__ == '__main__':
Expand Down
93 changes: 52 additions & 41 deletions tests/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,48 +25,58 @@ set(MODEL_IDS
# will be used to render each of the (relevant) test contexts into a golden file with
# the official Python jinja2 library. Then a test case will be created to run the C++
# minja implementation on the same template and context, and compare the output with the golden.
"abacusai/Fewshot-Metamath-OrcaVicuna-Mistral"
"bofenghuang/vigogne-2-70b-chat"
"deepseek-ai/deepseek-coder-33b-instruct"
"deepseek-ai/DeepSeek-Coder-V2-Instruct"
"deepseek-ai/DeepSeek-V2.5"
"indischepartij/MiniCPM-3B-OpenHermes-2.5-v2"
"meetkai/functionary-medium-v3.1"
"meetkai/functionary-medium-v3.2"
"microsoft/Phi-3-medium-4k-instruct"
"microsoft/Phi-3-mini-4k-instruct"
"microsoft/Phi-3-small-8k-instruct"
"microsoft/Phi-3.5-mini-instruct"
"microsoft/Phi-3.5-vision-instruct"
"mlabonne/AlphaMonarch-7B"
"NousResearch/Hermes-2-Pro-Llama-3-8B"
"NousResearch/Hermes-2-Pro-Mistral-7B"
"NousResearch/Hermes-3-Llama-3.1-70B"
"openchat/openchat-3.5-0106"
"OrionStarAI/Orion-14B-Chat"
"Qwen/Qwen2-7B-Instruct"
"Qwen/Qwen2-VL-7B-Instruct"
"Qwen/Qwen2.5-7B-Instruct"
"Qwen/Qwen2.5-Math-7B-Instruct"
"teknium/OpenHermes-2.5-Mistral-7B"
"TheBloke/FusionNet_34Bx2_MoE-AWQ"
)
#
# For Gated models, you'll need to run `huggingface-cli login` (and be granted access) to download their template.

abacusai/Fewshot-Metamath-OrcaVicuna-Mistral
bofenghuang/vigogne-2-70b-chat
CohereForAI/c4ai-command-r-plus # Gated
databricks/dbrx-instruct # Gated
deepseek-ai/deepseek-coder-33b-instruct
deepseek-ai/DeepSeek-Coder-V2-Instruct
deepseek-ai/DeepSeek-V2.5
dreamgen/WizardLM-2-7B
google/gemma-2-2b-it # Gated
google/gemma-7b-it # Gated
indischepartij/MiniCPM-3B-OpenHermes-2.5-v2
mattshumer/Reflection-Llama-3.1-70B
meetkai/functionary-medium-v3.1
meetkai/functionary-medium-v3.2
meta-llama/Llama-3.2-3B-Instruct # Gated
meta-llama/Meta-Llama-3.1-8B-Instruct # Gated
microsoft/Phi-3-medium-4k-instruct
microsoft/Phi-3-mini-4k-instruct
microsoft/Phi-3-small-8k-instruct
microsoft/Phi-3.5-mini-instruct
microsoft/Phi-3.5-vision-instruct
mistralai/Mistral-7B-Instruct-v0.2 # Gated
mistralai/Mistral-Large-Instruct-2407 # Gated
mistralai/Mistral-Large-Instruct-2411 # Gated
mistralai/Mistral-Nemo-Instruct-2407 # Gated
mistralai/Mixtral-8x7B-Instruct-v0.1 # Gated
mlabonne/AlphaMonarch-7B
NexaAIDev/Octopus-v2
NousResearch/Hermes-2-Pro-Llama-3-8B
NousResearch/Hermes-2-Pro-Mistral-7B
NousResearch/Hermes-3-Llama-3.1-70B
nvidia/Llama-3.1-Nemotron-70B-Instruct-HF
openchat/openchat-3.5-0106
OrionStarAI/Orion-14B-Chat
Qwen/Qwen2-7B-Instruct
Qwen/Qwen2-VL-7B-Instruct
Qwen/Qwen2.5-7B-Instruct
Qwen/Qwen2.5-Math-7B-Instruct
teknium/OpenHermes-2.5-Mistral-7B
TheBloke/FusionNet_34Bx2_MoE-AWQ

# Gated models: you will need to run `huggingface-cli login` (and be granted access) to download these
if (MINJA_TEST_GATED_MODELS)
list(APPEND MODEL_IDS
"meta-llama/Llama-3.2-3B-Instruct"
"meta-llama/Meta-Llama-3.1-8B-Instruct"
"google/gemma-7b-it"
"google/gemma-2-2b-it"
"mistralai/Mistral-7B-Instruct-v0.2"
"mistralai/Mixtral-8x7B-Instruct-v0.1"
"mistralai/Mistral-Large-Instruct-2407"
"mistralai/Mistral-Large-Instruct-2411"
"mistralai/Mistral-Nemo-Instruct-2407"
"CohereForAI/c4ai-command-r-plus"
)
endif()
# Broken, TODO:
# fireworks-ai/llama-3-firefunction-v2

# Can't find template(s), TODO:
# ai21labs/Jamba-v0.1
# apple/OpenELM-1_1B-Instruct
# xai-org/grok-1
)

# Create one test case for each {template, context} combination
file(GLOB CONTEXT_FILES "${CMAKE_SOURCE_DIR}/tests/contexts/*.json")
Expand All @@ -85,6 +95,7 @@ foreach(test_case ${CHAT_TEMPLATE_TEST_CASES})
list(GET test_args -1 last_arg)
string(REGEX REPLACE "^[^ ]+/([^ /]+)\\.[^.]+$" "\\1" test_name "${last_arg}")
add_test(NAME ${test_name} COMMAND $<TARGET_FILE:test-chat-template> ${test_args})
set_tests_properties(${test_name} PROPERTIES SKIP_RETURN_CODE 127)
endforeach()

if (MINJA_FUZZTEST_ENABLED)
Expand Down
9 changes: 8 additions & 1 deletion tests/test-chat-template.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,20 @@ int main(int argc, char *argv[]) {
std::string tmpl_file = argv[1];
std::string ctx_file = argv[2];
std::string golden_file = argv[3];

auto tmpl_str = read_file(tmpl_file);

if (ctx_file == "n/a")
{
std::cout << "# Skipping template: " << tmpl_file << "\n" << tmpl_str << std::endl;
return 127;
}

std::cout << "# Testing template: " << tmpl_file << std::endl
<< "# With context: " << ctx_file << std::endl
<< "# Against golden file: " << golden_file << std::endl
<< std::flush;

auto tmpl_str = read_file(tmpl_file);
auto ctx = json::parse(read_file(ctx_file));

minja::chat_template tmpl(
Expand Down
Loading