benchmarks.json

[
  {
    "id": "swe_v",
    "name": "SWE-Bench Verified",
    "category": "coding",
    "what_it_measures": "Real GitHub issue resolution, verified subset",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "low",
    "update_frequency": "monthly",
    "url": "https://scale.com/leaderboard",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Migrating to SWE-Bench Pro (uncontaminated). Current frontier ceiling ~81%.",
    "description": "Real GitHub issues solved by AI. Verified subset with less contamination risk.",
    "creator": "Scale AI + Princeton",
    "affiliation_risk": "low",
    "affiliation_note": "Independent verification by Scale AI. Princeton created tasks."
  },
  {
    "id": "swe_pro",
    "name": "SWE-Bench Pro",
    "category": "coding",
    "what_it_measures": "Real GitHub issues, multi-language, uncontaminated",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "monthly",
    "url": "https://scale.com/leaderboard",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Replacement for SWE-V. Better separation. Current frontier ceiling ~46%.",
    "description": "Harder coding tasks from real repos. Professional-level, less contamination.",
    "creator": "Scale AI",
    "affiliation_risk": "low",
    "affiliation_note": "Scale AI created and maintains."
  },
  {
    "id": "gpqa",
    "name": "GPQA Diamond",
    "category": "reasoning",
    "what_it_measures": "Graduate-level science questions (physics, chemistry, biology)",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://artificialanalysis.ai/models",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Frontier ceiling ~94%. Still meaningful separation.",
    "description": "Graduate-level science questions. PhD-verified, hard to guess.",
    "creator": "NYU/Cohere",
    "affiliation_risk": "medium",
    "affiliation_note": "Cohere co-authored. Possible bias toward Cohere models."
  },
  {
    "id": "hle",
    "name": "HLE (Humanity's Last Exam)",
    "category": "reasoning",
    "what_it_measures": "Hardest academic questions across all domains",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "monthly",
    "url": "https://lastexam.ai",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Deliberately hard. Current frontier ceiling ~42%. Contamination-resistant.",
    "description": "Hardest known reasoning benchmark. Expert questions no model aces.",
    "creator": "Scale AI + academics",
    "affiliation_risk": "low",
    "affiliation_note": "Multi-institution. Expert-sourced questions."
  },
  {
    "id": "arc_agi_2",
    "name": "ARC-AGI-2",
    "category": "reasoning",
    "what_it_measures": "Abstract reasoning, pure pattern generalization, no internet knowledge required",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://arcprize.org",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Pure LLMs scored 0% on ARC-AGI-1. ARC-AGI-2 harder. Frontier ceiling ~77%.",
    "description": "Abstract reasoning puzzles. Tests generalization, not memorization.",
    "creator": "ARC Prize Foundation",
    "affiliation_risk": "low",
    "affiliation_note": "Independent non-profit."
  },
  {
    "id": "tau2",
    "name": "Tau2-Bench",
    "category": "tools",
    "what_it_measures": "Multi-turn tool use in realistic scenarios",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://github.com/sierra-research/tau-bench",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Slight saturation at frontier (99.3%). Still meaningful for budget model comparison.",
    "description": "Multi-turn tool use. Tests agent reliability over many steps.",
    "creator": "Sierra Research",
    "affiliation_risk": "low",
    "affiliation_note": "No model affiliation."
  },
  {
    "id": "terminal_bench_2",
    "name": "Terminal-Bench 2",
    "category": "agentic",
    "what_it_measures": "CLI problem solving, agentic terminal tasks",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://github.com/harbor-framework/terminal-bench-2",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Frontier ceiling ~58%.",
    "description": "Agentic terminal coding. Real shell interactions and CLI tasks.",
    "creator": "Harbor Framework",
    "affiliation_risk": "low",
    "affiliation_note": "Open-source community project."
  },
  {
    "id": "gdpval",
    "name": "GDPval-AA",
    "category": "agentic",
    "what_it_measures": "Real-world work value: agentic tasks with measurable output",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://gdpval.pro/",
    "max_score": null,
    "lifecycle": "active",
    "notes": "Score is absolute value metric (~1500 range). Frontier ceiling ~1500+.",
    "description": "Economically valuable real-world task benchmark by Artificial Analysis.",
    "creator": "Artificial Analysis",
    "affiliation_risk": "low",
    "affiliation_note": "Independent benchmarking company."
  },
  {
    "id": "benchlm",
    "name": "BenchLM",
    "category": "general",
    "what_it_measures": "Aggregate benchmark score, community-driven",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "medium",
    "update_frequency": "monthly",
    "url": "https://benchlm.ai",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Composite score. Good for overall ranking comparison.",
    "description": "Aggregate benchmark across practical coding and reasoning tasks.",
    "creator": "BenchLM",
    "affiliation_risk": "low",
    "affiliation_note": "Independent benchmarking service."
  },
  {
    "id": "bfcl",
    "name": "BFCL V4",
    "category": "tools",
    "what_it_measures": "Function calling accuracy, multi-turn, parallel calls",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "low",
    "update_frequency": "monthly",
    "url": "https://gorilla.cs.berkeley.edu/leaderboard.html",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "V4 released 2026. Migrated from V3. Slight saturation at frontier.",
    "description": "Berkeley Function Calling Leaderboard. Tests tool/function calling accuracy.",
    "creator": "UC Berkeley",
    "affiliation_risk": "low",
    "affiliation_note": "Academic (Gorilla project)."
  },
  {
    "id": "lmarena",
    "name": "LMArena (ELO)",
    "category": "preference",
    "what_it_measures": "Human preference via blind pairwise comparison",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "high",
    "update_frequency": "continuous",
    "url": "https://lmarena.ai/leaderboard",
    "max_score": null,
    "lifecycle": "active",
    "notes": "ELO score. Volatile \u2014 sensitive to style and verbosity, not just quality.",
    "description": "Human preference ranking. Crowdsourced blind model comparisons.",
    "creator": "LMSYS/UC Berkeley",
    "affiliation_risk": "low",
    "affiliation_note": "Crowdsourced blind voting."
  },
  {
    "id": "simpleqa",
    "name": "SimpleQA",
    "category": "trust",
    "what_it_measures": "Factual accuracy, hallucination resistance",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://github.com/openai/simple-evals",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Hallucination resistance. Tests factual accuracy on simple questions.",
    "creator": "OpenAI",
    "affiliation_risk": "high",
    "affiliation_note": "Created by OpenAI. May favor GPT models."
  },
  {
    "id": "mmmu",
    "name": "MMMU",
    "category": "multimodal",
    "what_it_measures": "Multimodal understanding: images + text across disciplines",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://mmmu-benchmark.github.io",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Massive Multi-discipline Multimodal Understanding. Vision + reasoning.",
    "creator": "Academic (multi-institution)",
    "affiliation_risk": "low",
    "affiliation_note": "Large multi-institution effort."
  },
  {
    "id": "scicode",
    "name": "SciCode",
    "category": "coding",
    "what_it_measures": "Scientific coding: implementing algorithms from research papers",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://scicode-bench.github.io",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Scientific coding problems requiring domain knowledge.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Scientific coding benchmark."
  },
  {
    "id": "livebench",
    "name": "LiveBench",
    "category": "general",
    "what_it_measures": "Monthly fresh questions, objective scoring, no contamination",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "high",
    "update_frequency": "monthly",
    "url": "https://livebench.ai",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Monthly fresh evaluation. Contamination-free by design.",
    "creator": "Academic (CMU/Princeton)",
    "affiliation_risk": "low",
    "affiliation_note": "Monthly fresh eval. Contamination-free by design."
  },
  {
    "id": "browsecomp",
    "name": "BrowseComp",
    "category": "web",
    "what_it_measures": "Web understanding and retrieval comprehension",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://openai.com/index/browsecomp/",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "MiniMax M2.5 leads at 76.3%.",
    "description": "Web browsing comprehension by OpenAI.",
    "creator": "OpenAI",
    "affiliation_risk": "high",
    "affiliation_note": "OpenAI internal benchmark."
  },
  {
    "id": "ifbench",
    "name": "IFBench",
    "category": "instruction",
    "what_it_measures": "Instruction following: format, constraints, multi-condition",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://arxiv.org/abs/2502.09980",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Qwen3.5-397B leads at 78.8%.",
    "description": "Instruction following. Tests compliance with complex format constraints.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Research paper benchmark."
  },
  {
    "id": "grind",
    "name": "GRIND",
    "category": "reasoning",
    "what_it_measures": "Adaptive reasoning: increasingly hard problems",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://arxiv.org/abs/2501.06751",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Gemini 2.5 Pro leads at 82.1%.",
    "description": "Adaptive reasoning with adjustable difficulty.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Research paper benchmark."
  },
  {
    "id": "gaia",
    "name": "GAIA",
    "category": "agentic",
    "what_it_measures": "Multi-step general assistant tasks",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://huggingface.co/gaia-benchmark",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Slight saturation at frontier. Claude Sonnet 4.5 leads at 74.6%.",
    "description": "General AI Assistant benchmark. Real-world multi-step tasks.",
    "creator": "HuggingFace + Meta",
    "affiliation_risk": "medium",
    "affiliation_note": "Meta co-authored. HuggingFace hosts."
  },
  {
    "id": "humaneval",
    "name": "HumanEval",
    "category": "coding",
    "what_it_measures": "Python code generation from docstrings",
    "saturation": true,
    "contamination_risk": "high",
    "volatility": "low",
    "update_frequency": "N/A",
    "url": "https://github.com/openai/human-eval",
    "max_score": 100,
    "lifecycle": "dead",
    "notes": "All frontier models 90%+. No separation. Replaced by SWE-Bench.",
    "description": "Python code generation. Saturated \u2014 most models score 95%+.",
    "creator": "OpenAI",
    "affiliation_risk": "high",
    "affiliation_note": "OpenAI-created. Saturated and contaminated."
  },
  {
    "id": "mmlu",
    "name": "MMLU",
    "category": "general",
    "what_it_measures": "Multiple-choice academic knowledge across 57 subjects",
    "saturation": true,
    "contamination_risk": "high",
    "volatility": "low",
    "update_frequency": "N/A",
    "url": "https://arxiv.org/abs/2009.03300",
    "max_score": 100,
    "lifecycle": "dead",
    "notes": "All models 85%+. Saturated. Use MMLU-Pro instead.",
    "description": "Multi-task knowledge. Saturated and contaminated.",
    "creator": "Academic (UC Berkeley)",
    "affiliation_risk": "low",
    "affiliation_note": "Widely adopted academic benchmark."
  },
  {
    "id": "mmlu_pro",
    "name": "MMLU-Pro",
    "category": "general",
    "what_it_measures": "Harder version of MMLU with 10 choices, reasoning required",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://arxiv.org/abs/2406.01574",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Better than MMLU for separation. Frontier ceiling ~85%.",
    "description": "Professional-level multi-task knowledge. Harder than original MMLU.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Community improvement of MMLU."
  },
  {
    "id": "gsm8k",
    "name": "GSM8K",
    "category": "math",
    "what_it_measures": "Grade school math word problems",
    "saturation": true,
    "contamination_risk": "high",
    "volatility": "low",
    "update_frequency": "N/A",
    "url": "https://arxiv.org/abs/2110.14168",
    "max_score": 100,
    "lifecycle": "dead",
    "notes": "All frontier models 95%+. Use AIME 2025 instead.",
    "description": "Grade school math. Saturated \u2014 frontier models at 95%+.",
    "creator": "OpenAI",
    "affiliation_risk": "medium",
    "affiliation_note": "OpenAI-created dataset."
  },
  {
    "id": "aime_2025",
    "name": "AIME 2025",
    "category": "math",
    "what_it_measures": "American Invitational Mathematics Examination 2025",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "annual",
    "url": "https://artofproblemsolving.com/wiki/index.php/AIME",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "GPT-5.4 scored 100%. Gemini 3.1 Pro 92%.",
    "description": "Math competition problems. American Invitational Mathematics Exam.",
    "creator": "Mathematical Association of America",
    "affiliation_risk": "low",
    "affiliation_note": "Independent math competition."
  },
  {
    "id": "hellaswag",
    "name": "HellaSwag",
    "category": "general",
    "what_it_measures": "Commonsense NLI",
    "saturation": true,
    "contamination_risk": "high",
    "volatility": "low",
    "update_frequency": "N/A",
    "url": "https://arxiv.org/abs/1905.07830",
    "max_score": 100,
    "lifecycle": "dead",
    "notes": "All models 90%+. No meaningful separation.",
    "description": "Common sense reasoning. Saturated at 95%+.",
    "creator": "Academic (UW)",
    "affiliation_risk": "low",
    "affiliation_note": "University of Washington."
  },
  {
    "id": "bbh",
    "name": "BBH (BIG-Bench Hard)",
    "category": "reasoning",
    "what_it_measures": "Hard subset of BIG-Bench tasks",
    "saturation": true,
    "contamination_risk": "high",
    "volatility": "low",
    "update_frequency": "N/A",
    "url": "https://arxiv.org/abs/2210.09261",
    "max_score": 100,
    "lifecycle": "dead",
    "notes": "All frontier models 90%+. Saturated.",
    "description": "BIG-Bench Hard. 23 challenging tasks. Largely saturated.",
    "creator": "Google + Academic",
    "affiliation_risk": "medium",
    "affiliation_note": "Google co-authored BIG-Bench."
  },
  {
    "id": "math_base",
    "name": "MATH (base)",
    "category": "math",
    "what_it_measures": "Competition math problems",
    "saturation": true,
    "contamination_risk": "high",
    "volatility": "low",
    "update_frequency": "N/A",
    "url": "https://arxiv.org/abs/2103.03874",
    "max_score": 100,
    "lifecycle": "dead",
    "notes": "All frontier models 90%+. Use AIME or HLE for math separation.",
    "description": "Competition math (base). Superseded by AIME 2025.",
    "creator": "Academic (UC Berkeley)",
    "affiliation_risk": "low",
    "affiliation_note": "MATH dataset."
  },
  {
    "id": "drop",
    "name": "DROP",
    "category": "reasoning",
    "what_it_measures": "Discrete reasoning over paragraphs",
    "saturation": true,
    "contamination_risk": "high",
    "volatility": "low",
    "update_frequency": "N/A",
    "url": "https://arxiv.org/abs/1903.00161",
    "max_score": 100,
    "lifecycle": "dead",
    "notes": "Saturated.",
    "description": "Discrete reasoning over paragraphs. Saturated.",
    "creator": "Academic (Allen AI)",
    "affiliation_risk": "low",
    "affiliation_note": "Allen Institute for AI."
  },
  {
    "id": "mgsm",
    "name": "MGSM",
    "category": "multilingual",
    "what_it_measures": "Multilingual grade school math",
    "saturation": true,
    "contamination_risk": "high",
    "volatility": "low",
    "update_frequency": "N/A",
    "url": "https://arxiv.org/abs/2210.03057",
    "max_score": 100,
    "lifecycle": "saturated",
    "notes": "Saturated at frontier. GLM-5 leads at 90% but no separation.",
    "description": "Multilingual grade school math. Tests reasoning across languages.",
    "creator": "Google",
    "affiliation_risk": "medium",
    "affiliation_note": "Google-created multilingual math."
  },
  {
    "id": "medqa",
    "name": "MedQA (USMLE)",
    "category": "medical",
    "what_it_measures": "USMLE-style medical licensing exam questions",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "low",
    "update_frequency": "annual",
    "url": "https://arxiv.org/abs/2009.13081",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Frontier 85-95%. Weakly saturating. Use for medical domain benchmarking.",
    "description": "Medical board exam questions (USMLE). Tests clinical reasoning.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Medical board exam questions. No model affiliation."
  },
  {
    "id": "pubmedqa",
    "name": "PubMedQA",
    "category": "medical",
    "what_it_measures": "Biomedical literature question answering",
    "saturation": false,
    "contamination_risk": "medium",
    "volatility": "low",
    "update_frequency": "annual",
    "url": "https://arxiv.org/abs/1909.06146",
    "max_score": 100,
    "lifecycle": "planned",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Biomedical literature question answering from PubMed abstracts.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Biomedical QA from PubMed."
  },
  {
    "id": "scale_seal",
    "name": "Scale SEAL",
    "category": "domain",
    "what_it_measures": "Domain-specific: legal, finance, coding evaluated by experts",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://scale.com/leaderboard",
    "max_score": 100,
    "lifecycle": "planned",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Scale AI domain evaluation. Legal and finance benchmarks.",
    "creator": "Scale AI",
    "affiliation_risk": "medium",
    "affiliation_note": "Scale AI paid by model providers for eval."
  },
  {
    "id": "throughput_tps",
    "name": "Throughput (tokens/sec)",
    "category": "infrastructure",
    "what_it_measures": "Sustained token generation speed in tokens per second",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://artificialanalysis.ai/models",
    "max_score": null,
    "lifecycle": "active",
    "notes": "Higher = faster. Llama 4 Scout 2600 tps, Mercury 2 872 tps.",
    "description": "Tokens per second output speed benchmark.",
    "creator": "Artificial Analysis",
    "affiliation_risk": "low",
    "affiliation_note": "Objective speed measurement."
  },
  {
    "id": "tool_error_rate",
    "name": "Tool Call Error Rate",
    "category": "tools",
    "what_it_measures": "Percentage of tool calls that result in errors or malformed calls",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://gorilla.cs.berkeley.edu/leaderboard",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Lower = better. GLM-5-Turbo 0.67% error rate.",
    "description": "Rate of errors in tool/function calls during execution.",
    "creator": "UC Berkeley",
    "affiliation_risk": "low",
    "affiliation_note": "Academic (Gorilla project)."
  },
  {
    "id": "scraper_code_zyte",
    "name": "Zyte Scraper Code Quality",
    "category": "web_scraping",
    "what_it_measures": "ROUGE-1 extraction quality for web scraper code generation",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "annual",
    "url": "https://zyte.com/blog",
    "max_score": 1.0,
    "lifecycle": "active",
    "notes": "Feb 2026 benchmark. Sonnet 4.6 leads at 0.835.",
    "description": "Web scraper code generation quality (ROUGE-1 score).",
    "creator": "Zyte",
    "affiliation_risk": "medium",
    "affiliation_note": "Zyte sells scraping tools."
  },
  {
    "id": "vending_b2",
    "name": "VendingBench 2",
    "category": "agentic",
    "what_it_measures": "Long-horizon agentic task completion in vending machine scenario",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://github.com/sierra-research/tau2-bench",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Long-horizon multi-step tasks. Claude Opus 4.5 leads.",
    "description": "Retail/airline/telecom customer-service agent benchmark. Multi-turn tasks.",
    "creator": "Sierra Research",
    "affiliation_risk": "low",
    "affiliation_note": "Same team as Tau-Bench."
  },
  {
    "id": "human_pref",
    "name": "Human Preference (writing)",
    "category": "preference",
    "what_it_measures": "Human preference for writing quality via pairwise evaluation",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "high",
    "update_frequency": "quarterly",
    "url": "https://lmarena.ai/leaderboard",
    "max_score": null,
    "lifecycle": "active",
    "notes": "Subjective writing quality. Claude Opus 4.6 and GPT-5.4 lead.",
    "description": "Writing naturalness rated by humans in blind comparisons.",
    "creator": "LMSYS/UC Berkeley",
    "affiliation_risk": "low",
    "affiliation_note": "Crowdsourced blind preference."
  },
  {
    "id": "long_context",
    "name": "Long Context (>200K)",
    "category": "general",
    "what_it_measures": "Effective context utilization beyond 200K tokens",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://arxiv.org/abs/2404.06654",
    "max_score": null,
    "lifecycle": "planned",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "RULER benchmark. Tests retrieval and reasoning at >200K context.",
    "creator": "Academic (RULER)",
    "affiliation_risk": "low",
    "affiliation_note": "Independent context length eval."
  },
  {
    "id": "browseruse",
    "name": "BrowserUse",
    "category": "web",
    "what_it_measures": "Browser automation: click, fill, navigate, extract",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://browser-use.com",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Real-world browser automation. Claude Sonnet 4.6 leads.",
    "description": "Automated browser interaction and web task completion.",
    "creator": "Browser Use",
    "affiliation_risk": "medium",
    "affiliation_note": "Browser Use startup benchmark."
  },
  {
    "id": "jsdeobs",
    "name": "JS Deobfuscation",
    "category": "reverse_engineering",
    "what_it_measures": "Accuracy of JavaScript deobfuscation and code recovery",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "annual",
    "url": "https://github.com/nickcano/llm-deobfuscation-benchmark",
    "max_score": 1.0,
    "lifecycle": "planned",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "JavaScript deobfuscation. Security research benchmark.",
    "creator": "Academic (CCS 2025)",
    "affiliation_risk": "low",
    "affiliation_note": "Security conference paper."
  },
  {
    "id": "promon",
    "name": "ProMon (Assembly RE)",
    "category": "reverse_engineering",
    "what_it_measures": "Assembly/binary deobfuscation and reverse engineering",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "annual",
    "url": "https://promon.co/security-news/",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Assembly deobfuscation. Claude Sonnet 4.6 leads.",
    "description": "Assembly deobfuscation. Binary reverse engineering tasks.",
    "creator": "Promon",
    "affiliation_risk": "medium",
    "affiliation_note": "Promon sells app protection."
  },
  {
    "id": "analytics_accuracy",
    "name": "Analytics Accuracy (Anamap LLM)",
    "category": "data_analysis",
    "what_it_measures": "Accuracy of data analysis tasks: marketing, financial, product analytics",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "annual",
    "url": "https://anamaps.com",
    "max_score": 100,
    "lifecycle": "planned",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Data analysis and interpretation accuracy.",
    "creator": "Anamap",
    "affiliation_risk": "medium",
    "affiliation_note": "Anamap product benchmark."
  },
  {
    "id": "wer_clean",
    "name": "WER Clean Speech",
    "category": "audio",
    "what_it_measures": "Word Error Rate on clean speech (studio/phone quality audio)",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://github.com/openai/whisper",
    "max_score": null,
    "lifecycle": "active",
    "notes": "Lower = better. Best: Soniox 3.2%, Deepgram Nova-3 3.5%, Whisper large-v3 4.2%.",
    "description": "Word Error Rate on clean speech. Transcription accuracy.",
    "creator": "OpenAI",
    "affiliation_risk": "medium",
    "affiliation_note": "Test sets from Whisper paper."
  },
  {
    "id": "wer_noisy",
    "name": "WER Noisy Speech",
    "category": "audio",
    "what_it_measures": "Word Error Rate on noisy/real-world audio conditions",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://arxiv.org/abs/2212.04356",
    "max_score": null,
    "lifecycle": "planned",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Word Error Rate on noisy speech. Tests robustness to background noise.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Standard speech recognition metric."
  },
  {
    "id": "diarization_der",
    "name": "Diarization Error Rate (DER)",
    "category": "audio",
    "what_it_measures": "Speaker diarization accuracy: who spoke when",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "quarterly",
    "url": "https://github.com/pyannote/pyannote-audio",
    "max_score": null,
    "lifecycle": "planned",
    "notes": "Benchmark tracked, scores not yet collected.",
    "description": "Diarization Error Rate. Speaker identification accuracy.",
    "creator": "Academic (pyannote)",
    "affiliation_risk": "low",
    "affiliation_note": "Open-source standard."
  },
  {
    "id": "video_mme",
    "name": "Video-MME",
    "category": "multimodal",
    "what_it_measures": "Video multi-modal evaluation: comprehension, QA, temporal reasoning",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "medium",
    "update_frequency": "quarterly",
    "url": "https://video-mme.github.io",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Native video input required. Gemini 3.1 Pro leads.",
    "description": "Video understanding. Multi-modal evaluation across video content.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Multi-institution video benchmark."
  },
  {
    "id": "fuzzing_labs",
    "name": "Fuzzing Labs (APK RE)",
    "category": "reverse_engineering",
    "what_it_measures": "Android APK reverse engineering accuracy",
    "saturation": false,
    "contamination_risk": "low",
    "volatility": "low",
    "update_frequency": "annual",
    "url": "https://github.com/nickcano/llm-apk-analysis",
    "max_score": 100,
    "lifecycle": "active",
    "notes": "Android binary analysis. Claude Sonnet 4.6 leads.",
    "description": "Android APK reverse engineering and analysis.",
    "creator": "FuzzingLabs",
    "affiliation_risk": "medium",
    "affiliation_note": "Security company benchmark."
  },
  {
    "id": "code_review_bench",
    "name": "CodeReviewBench",
    "category": "coding",
    "lifecycle": "active",
    "volatility": "medium",
    "contamination_risk": "low",
    "url": "https://codereviewbench.com/",
    "description": "How well LLMs identify bugs, security issues, and style problems in real code review.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Independent code review eval."
  },
  {
    "id": "writing_bench",
    "name": "WritingBench",
    "category": "writing",
    "lifecycle": "active",
    "volatility": "medium",
    "contamination_risk": "low",
    "url": "https://arxiv.org/abs/2503.05244v3",
    "description": "Academic and professional writing quality evaluation.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Academic writing quality eval."
  },
  {
    "id": "some_bench",
    "name": "SoMe",
    "category": "agentic",
    "lifecycle": "active",
    "volatility": "medium",
    "contamination_risk": "low",
    "url": "https://arxiv.org/html/2512.14720v1",
    "description": "LLM-based social media agent benchmark. Tests autonomous social interaction.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Social media agent benchmark."
  },
  {
    "id": "turing_bench",
    "name": "TuringBench",
    "category": "trust",
    "lifecycle": "active",
    "volatility": "medium",
    "contamination_risk": "low",
    "url": "https://turingbench.ist.psu.edu/",
    "description": "Human vs machine text detection and authorship attribution.",
    "creator": "Academic (Penn State)",
    "affiliation_risk": "low",
    "affiliation_note": "Human vs AI detection research."
  },
  {
    "id": "pmc_patients",
    "name": "PMC-Patients",
    "category": "medical",
    "lifecycle": "active",
    "volatility": "low",
    "contamination_risk": "low",
    "url": "https://pmc-patients.github.io/",
    "description": "Retrieval-based clinical decision support. Patient-to-article and patient-to-patient retrieval.",
    "creator": "Academic",
    "affiliation_risk": "low",
    "affiliation_note": "Clinical decision support benchmark."
  },
  {
    "id": "bin_metric",
    "name": "BinMetric",
    "category": "reverse_engineering",
    "lifecycle": "active",
    "volatility": "low",
    "contamination_risk": "low",
    "url": "https://arxiv.org/html/2505.07360v1",
    "description": "Assembly understanding, function recognition, vulnerability detection.",
    "creator": "Academic (IJCAI 2025)",
    "affiliation_risk": "low",
    "affiliation_note": "Binary RE academic benchmark."
  },
  {
    "id": "mlx_bench",
    "name": "MLX On-Device Bench",
    "category": "local",
    "lifecycle": "active",
    "volatility": "medium",
    "contamination_risk": "low",
    "url": "https://arxiv.org/abs/2510.18921",
    "description": "LLM inference speed and efficiency on Apple Silicon with MLX.",
    "creator": "Apple/Academic",
    "affiliation_risk": "medium",
    "affiliation_note": "Apple co-authored. Tests Apple Silicon."
  }
]