From 86dd532948836fac08355cb535b8ff53aa64b9c7 Mon Sep 17 00:00:00 2001 From: Jacob Ioffe Date: Tue, 10 Mar 2026 23:36:29 +0000 Subject: [PATCH] docs: align benchmarking guide with harness defaults - update active defaults to match test_configs.yaml values - nest compose profiles under compose and set sparse false - correct bo20 infographics in dataset config and table Signed-off-by: Jacob Ioffe (cherry picked from commit 85a919232f80e37919486c0121d10edf53d4272f) --- docs/docs/extraction/benchmarking.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/docs/extraction/benchmarking.md b/docs/docs/extraction/benchmarking.md index 62abbf302..246b3788d 100644 --- a/docs/docs/extraction/benchmarking.md +++ b/docs/docs/extraction/benchmarking.md @@ -68,16 +68,18 @@ active: test_name: null # Auto-generated if null # API Configuration - api_version: v1 # v1 or v2 + api_version: v2 # v1 or v2 pdf_split_page_count: null # V2 only: pages per chunk (null = default 32) # Infrastructure hostname: localhost readiness_timeout: 600 - profiles: [retrieval] + compose: + profiles: + - retrieval # Runtime - sparse: true + sparse: false gpu_search: false embedding_model: auto @@ -123,7 +125,7 @@ datasets: extract_tables: true extract_charts: true extract_images: true - extract_infographics: false + extract_infographics: true recall_dataset: null # bo20 does not have recall earnings: @@ -159,7 +161,7 @@ uv run nemo-retriever-bench --case=e2e --dataset=/custom/path |---------|------|--------|--------|--------|--------------|--------| | `bo767` | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | | `earnings` | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | -| `bo20` | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| `bo20` | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | | `financebench` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | `single` | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |