[
  {
    "id": "arena-elo",
    "name": "Chatbot Arena Elo",
    "shortName": "Arena",
    "category": "preference",
    "modality": "text",
    "description": "Crowdsourced pairwise human preference rankings of LLM responses. Higher Elo means more frequently preferred by users.",
    "higherIsBetter": true,
    "scoreUnit": "Elo",
    "lastRefreshed": "2026-04-01",
    "contaminationRisk": "low",
    "contaminationNote": "New user prompts arrive continuously; there is no fixed question set to memorise."
  },
  {
    "id": "mmlu-pro",
    "name": "MMLU Pro",
    "shortName": "MMLU-Pro",
    "category": "knowledge",
    "modality": "text",
    "description": "Harder version of MMLU testing knowledge across 57 academic subjects; reduces guessing-friendly answers.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2024-05-15",
    "contaminationRisk": "high",
    "contaminationNote": "Public static set; frontier models routinely clear 85%+ and training corpora likely include near-duplicates."
  },
  {
    "id": "gpqa-diamond",
    "name": "GPQA Diamond",
    "shortName": "GPQA",
    "category": "reasoning",
    "modality": "text",
    "description": "Graduate-level Google-proof Q&A in physics, chemistry, and biology. Diamond subset is the hardest tier with PhD-validated answers.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2023-11-01",
    "contaminationRisk": "medium",
    "contaminationNote": "Static set but expert-level; frontier reasoning models are now approaching 80%, so discriminative power is eroding."
  },
  {
    "id": "math-500",
    "name": "MATH-500",
    "shortName": "MATH",
    "category": "math",
    "modality": "text",
    "description": "500 high-school competition math problems requiring multi-step solutions. Scored on final-answer correctness.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2021-03-01",
    "contaminationRisk": "saturated",
    "contaminationNote": "Frontier reasoning models score 95%+; essentially a ceiling test now."
  },
  {
    "id": "aime-2024",
    "name": "AIME 2024",
    "shortName": "AIME",
    "category": "math",
    "modality": "text",
    "description": "American Invitational Mathematics Examination 2024 problems. Three-digit integer answers; very hard for non-reasoning models.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2024-02-01",
    "contaminationRisk": "high",
    "contaminationNote": "Problems are publicly released each year; training corpora crawled after Feb 2024 likely include solutions."
  },
  {
    "id": "humaneval",
    "name": "HumanEval",
    "shortName": "HumanEval",
    "category": "coding",
    "modality": "text",
    "description": "164 hand-written Python programming problems scored by passing unit tests. Saturated for frontier models.",
    "higherIsBetter": true,
    "scoreUnit": "% pass@1",
    "lastRefreshed": "2021-07-01",
    "contaminationRisk": "saturated",
    "contaminationNote": "Public since 2021; every frontier model scores 90%+. Included for historical comparability only."
  },
  {
    "id": "livecodebench",
    "name": "LiveCodeBench",
    "shortName": "LiveCB",
    "category": "coding",
    "modality": "text",
    "description": "Continuously refreshed coding benchmark drawing from LeetCode, AtCoder, and Codeforces; reduces benchmark contamination.",
    "higherIsBetter": true,
    "scoreUnit": "% pass@1",
    "lastRefreshed": "2026-04-01",
    "contaminationRisk": "low",
    "contaminationNote": "Rolling windows of fresh competitive-programming problems; problems from after a model's training cutoff are reported separately."
  },
  {
    "id": "swe-bench-verified",
    "name": "SWE-bench Verified",
    "shortName": "SWE-bench",
    "category": "agentic",
    "modality": "text",
    "description": "Real GitHub issues solved end-to-end. Verified subset is a 500-task human-validated slice of SWE-bench.",
    "higherIsBetter": true,
    "scoreUnit": "% resolved",
    "lastRefreshed": "2024-08-15",
    "contaminationRisk": "medium",
    "contaminationNote": "Issues and patches are scraped from public GitHub, but solving requires real tool use so memorisation is insufficient."
  },
  {
    "id": "ifeval",
    "name": "IFEval",
    "shortName": "IFEval",
    "category": "instruction-following",
    "modality": "text",
    "description": "Verifiable instruction-following benchmark; 25 categories of strict formatting / structural directives.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2023-11-01",
    "contaminationRisk": "medium",
    "contaminationNote": "Prompts are programmatically generated from templates; template overlap with training data is possible but verification is objective."
  },
  {
    "id": "mmmu",
    "name": "MMMU",
    "shortName": "MMMU",
    "category": "vision",
    "modality": "vision",
    "description": "Massive Multi-discipline Multimodal Understanding; college-exam level questions with images across 30+ subjects.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2024-03-15",
    "contaminationRisk": "medium",
    "contaminationNote": "Public static set; rigorously validated but increasingly solvable by frontier multimodal models."
  },
  {
    "id": "mmmu-pro",
    "name": "MMMU-Pro",
    "shortName": "MMMU-Pro",
    "category": "vision",
    "modality": "vision",
    "description": "Harder MMMU variant: text-only-answerable questions removed, ten answer options instead of four, and a vision-only setting where the question is embedded in the image.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2024-09-04",
    "contaminationRisk": "medium",
    "contaminationNote": "Public static set, but filtered to defeat text-only shortcuts and still well below the human-expert ceiling."
  },
  {
    "id": "mathvista",
    "name": "MathVista",
    "shortName": "MathVista",
    "category": "vision",
    "modality": "vision",
    "description": "Math reasoning over visual contexts (charts, figures, geometry).",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2023-10-01",
    "contaminationRisk": "medium",
    "contaminationNote": "Public static set; frontier models clear 70%."
  },
  {
    "id": "long-context-ruler",
    "name": "RULER 128k",
    "shortName": "RULER",
    "category": "long-context",
    "modality": "text",
    "description": "Long-context retrieval and reasoning suite. We report the 128k token effective-context score.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2024-08-01",
    "contaminationRisk": "low",
    "contaminationNote": "Questions are procedurally generated at evaluation time from the needle/haystack template; specific prompts are not reusable."
  },
  {
    "id": "image-arena-elo",
    "name": "Image Arena Elo",
    "shortName": "Img Arena",
    "category": "image-gen",
    "modality": "image-gen",
    "description": "Crowdsourced pairwise human preference for image generation models. Users vote on anonymised side-by-side generations; scores are a standard Elo over those votes.",
    "higherIsBetter": true,
    "scoreUnit": "Elo",
    "lastRefreshed": "2026-04-01",
    "contaminationRisk": "low",
    "contaminationNote": "Crowdsourced preference votes; no fixed prompt set."
  },
  {
    "id": "image-prompt-adherence",
    "name": "Prompt Adherence",
    "shortName": "Prompt Fid.",
    "category": "image-gen",
    "modality": "image-gen",
    "description": "How well the generated image matches the textual prompt as evaluated by human raters.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-04-01",
    "contaminationRisk": "low",
    "contaminationNote": "Fresh human-rated prompts; no static question set."
  },
  {
    "id": "tokens-per-second",
    "name": "Output Speed",
    "shortName": "Speed",
    "category": "performance",
    "modality": "text",
    "description": "Output speed in tokens per second: the 14-day median of Artificial Analysis's daily headline figure for each model, recorded nightly. Models it does not cover fall back to a dated manual figure. Higher is faster.",
    "higherIsBetter": true,
    "scoreUnit": "tok/s",
    "lastRefreshed": "2026-11-16",
    "contaminationRisk": "n/a",
    "contaminationNote": "Performance metric; contamination is not applicable."
  },
  {
    "id": "time-to-first-token",
    "name": "Time to First Token",
    "shortName": "TTFT",
    "category": "performance",
    "modality": "text",
    "description": "Time from request to first streamed token in milliseconds: the 14-day median of Artificial Analysis's daily headline figure for each model. Lower is snappier; reasoning models are penalised here because they think before talking.",
    "higherIsBetter": false,
    "scoreUnit": "ms",
    "lastRefreshed": "2026-11-16",
    "contaminationRisk": "n/a",
    "contaminationNote": "Performance metric; contamination is not applicable."
  },
  {
    "id": "livebench-global",
    "name": "Rolling Contamination-Controlled Average",
    "shortName": "Rolling Avg",
    "category": "general",
    "modality": "text",
    "description": "Contamination-controlled average across every task category in the latest rolling release (e.g. reasoning, coding, data analysis, research, browsing, long-horizon tasks). Questions are rotated with each release and graded against objective ground truth, removing the need for LLM-as-judge scoring. Only models with a score in every category get an average.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-06-25",
    "contaminationRisk": "low",
    "contaminationNote": "A fresh question set is released every six months and the latest batch is held back from the public \u2014 currently the strongest public signal for contamination-adjusted quality."
  },
  {
    "id": "livebench-data-analysis",
    "name": "Rolling Data Analysis",
    "shortName": "Data Analysis",
    "category": "data-analysis",
    "modality": "text",
    "description": "Rolling contamination-controlled data-analysis evaluation. Table comprehension, CSV / spreadsheet reasoning, SQL-style joins, and chart interpretation. Refreshed every six months with new tables and questions to minimise contamination.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-06-25",
    "contaminationRisk": "low",
    "contaminationNote": "Fresh tables and questions each release; no stable question set to memorise."
  },
  {
    "id": "frontiermath-t13",
    "name": "FrontierMath Tiers 1-3",
    "shortName": "FrontierMath",
    "category": "math",
    "modality": "text",
    "description": "Mathematical research problems spanning analysis, algebra, combinatorics and number theory. Tiers 1-3 are progressively harder; even frontier reasoning models only solve a small fraction. The hardest publicly reported benchmark for general mathematical reasoning.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2026-04-23",
    "contaminationRisk": "low",
    "contaminationNote": "Closed problem set held by Epoch AI; questions are not publicly released, so contamination risk is minimal."
  },
  {
    "id": "simpleqa-verified",
    "name": "SimpleQA Verified",
    "shortName": "SimpleQA",
    "category": "knowledge",
    "modality": "text",
    "description": "A human-validated factuality benchmark of short factual questions whose answers can be checked against a single ground truth. Penalises hallucinations by scoring confidently-wrong answers below abstentions.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2026-04-23",
    "contaminationRisk": "low",
    "contaminationNote": "Validated subset rerun by Epoch AI with held-out grading prompts; contamination risk is low."
  },
  {
    "id": "otis-mock-aime",
    "name": "OTIS Mock AIME 2024-2025",
    "shortName": "OTIS AIME",
    "category": "math",
    "modality": "text",
    "description": "AIME-style competition problems written specifically for the OTIS mock contest, then run as an evaluation by Epoch AI. Closer in spirit to the public AIME but with novel problems unlikely to appear in training data.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2026-04-23",
    "contaminationRisk": "low",
    "contaminationNote": "Novel problems written for the OTIS mock contest; less likely to be in training data than the official AIME."
  },
  {
    "id": "humanitys-last-exam",
    "name": "Humanity's Last Exam",
    "shortName": "HLE",
    "category": "reasoning",
    "modality": "text",
    "description": "A challenging multi-disciplinary exam aggregating expert-written questions from across academic fields. Designed to discriminate at the very top of the capability range when MMLU-style tests saturate.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2026-04-08",
    "contaminationRisk": "low",
    "contaminationNote": "Expert-written questions with limited public exposure; held to be the strongest open signal at the frontier."
  },
  {
    "id": "arc-agi-2",
    "name": "ARC-AGI 2",
    "shortName": "ARC-AGI 2",
    "category": "reasoning",
    "modality": "text",
    "description": "Second-generation ARC challenge testing fluid reasoning over abstract visual puzzles. Resists training-data memorisation by construction: each puzzle is novel and solutions require multi-step pattern induction. Frontier models are only just starting to score above chance on the harder tier.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2026-04-23",
    "contaminationRisk": "low",
    "contaminationNote": "Procedurally generated puzzles with novel patterns; very hard to contaminate."
  },
  {
    "id": "aider-polyglot",
    "name": "Aider Polyglot",
    "shortName": "Aider",
    "category": "coding",
    "modality": "text",
    "description": "Real-world refactoring and bug-fix tasks across multiple programming languages, scored by whether the model produces a passing patch in Aider's edit format. Tests practical coding ability beyond single-file generation; harder than HumanEval and not yet saturated.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2025-08-05",
    "contaminationRisk": "low",
    "contaminationNote": "Tasks rotate frequently and require a specific edit format; modest contamination risk."
  },
  {
    "id": "terminal-bench-2",
    "name": "Terminal-Bench 2",
    "shortName": "TermBench 2",
    "category": "agentic",
    "modality": "text",
    "description": "Long-horizon shell-and-filesystem tasks executed in a sandboxed terminal, scored by whether the agent's final state matches a target state. Tests practical tool-using ability for everyday devops and data-wrangling work; one of the hardest agentic benchmarks today.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2026-04-23",
    "contaminationRisk": "low",
    "contaminationNote": "Procedurally rotating task set with executed verification; very hard to game by memorisation."
  },
  {
    "id": "frontier-composite",
    "name": "Frontier Composite",
    "shortName": "Frontier",
    "category": "composite",
    "modality": "text",
    "description": "Saturation-resistant composite capability score stitched together from ~40 underlying benchmarks using Item Response Theory. Each benchmark is weighted by its fitted difficulty and discriminative slope, so doing well on hard, contamination-resistant evals (FrontierMath, ARC-AGI 2, Humanity's Last Exam) moves the score and saturated benchmarks contribute almost nothing. Imported per-model from Epoch AI's published index; we anchor it to the same min-max scale we use for every other benchmark so it's directly weightable in scenarios.",
    "higherIsBetter": true,
    "scoreUnit": "ECI",
    "lastRefreshed": "2026-04-23",
    "contaminationRisk": "low",
    "contaminationNote": "Aggregates many contamination-resistant benchmarks through IRT-fitted difficulty weights; saturated benchmarks contribute negligibly."
  },
  {
    "id": "output-stability",
    "name": "Output Stability",
    "shortName": "Stability",
    "category": "reliability",
    "modality": "text",
    "description": "How consistent the model's outputs are across repeated runs of the same task. Higher means lower variance, fewer occasional hallucinations under identical inputs. Useful for production loops that need reproducible behaviour.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-05-01",
    "contaminationRisk": "n/a",
    "contaminationNote": "Operational metric measured against a rolling task suite; not a quality benchmark."
  },
  {
    "id": "format-adherence",
    "name": "Format Adherence",
    "shortName": "Format",
    "category": "reliability",
    "modality": "text",
    "description": "How reliably the model produces output in the requested format (JSON schemas, markdown structures, exact-string responses). Pairs well with IFEval but reflects how the deployed API is behaving day to day rather than how a frozen test set scores.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-05-01",
    "contaminationRisk": "n/a",
    "contaminationNote": "Operational metric; not a quality benchmark."
  },
  {
    "id": "recovery-rate",
    "name": "Recovery Rate",
    "shortName": "Recovery",
    "category": "reliability",
    "modality": "text",
    "description": "How often the model self-corrects after producing an incorrect intermediate step (debugging axis upstream). Critical for agentic loops that depend on the model noticing and repairing its own mistakes rather than barrelling forward.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-05-01",
    "contaminationRisk": "n/a",
    "contaminationNote": "Operational metric; not a quality benchmark."
  },
  {
    "id": "api-success-rate",
    "name": "API Success Rate",
    "shortName": "API OK",
    "category": "reliability",
    "modality": "text",
    "description": "Uptime of the model maker's own API (standard tier), or the median across hosts when the maker does not serve it: the 14-day median of the public routing catalog's 24-hour uptime, recorded nightly.",
    "higherIsBetter": true,
    "scoreUnit": "%",
    "lastRefreshed": "2026-11-16",
    "contaminationRisk": "n/a",
    "contaminationNote": "Operational metric; contamination is not applicable."
  },
  {
    "id": "safety-handling",
    "name": "Safety Handling",
    "shortName": "Safety",
    "category": "reliability",
    "modality": "text",
    "description": "How well the model handles safety-sensitive prompts without false-refusing benign requests or producing unsafe output. The upstream signal does not separate refusal counts from substantive content-safety behaviour, so this single axis covers both.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-05-01",
    "contaminationRisk": "n/a",
    "contaminationNote": "Operational metric; not a quality benchmark."
  },
  {
    "id": "decision-index",
    "name": "Decision Index",
    "shortName": "Decision",
    "category": "decision",
    "modality": "text",
    "description": "Overall score of a decision model on a suite of single-choice tasks (knowledge, language, retrieval, tool use, arts), using the suite's own weighting rather than a plain average of the five categories. Measures how often the model picks the right answer from a fixed set; not comparable with chat-model benchmarks.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-10-05",
    "contaminationRisk": "medium",
    "contaminationNote": "Self-reported by a model maker on its own card. The card says benchmark records were held out of training by ID and checksum."
  },
  {
    "id": "decision-index-knowledge",
    "name": "Decision Index \u00b7 Knowledge",
    "shortName": "DI Know",
    "category": "decision",
    "modality": "text",
    "description": "Knowledge category of the Decision Index.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-10-05",
    "contaminationRisk": "medium"
  },
  {
    "id": "decision-index-language",
    "name": "Decision Index \u00b7 Language",
    "shortName": "DI Lang",
    "category": "decision",
    "modality": "text",
    "description": "Language category of the Decision Index.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-10-05",
    "contaminationRisk": "medium"
  },
  {
    "id": "decision-index-retrieval",
    "name": "Decision Index \u00b7 Retrieval",
    "shortName": "DI Retr",
    "category": "decision",
    "modality": "text",
    "description": "Retrieval category of the Decision Index.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-10-05",
    "contaminationRisk": "medium"
  },
  {
    "id": "decision-index-tools",
    "name": "Decision Index \u00b7 Tools",
    "shortName": "DI Tools",
    "category": "decision",
    "modality": "text",
    "description": "Tool-use category of the Decision Index.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-10-05",
    "contaminationRisk": "medium"
  },
  {
    "id": "decision-index-arts",
    "name": "Decision Index \u00b7 Arts",
    "shortName": "DI Arts",
    "category": "decision",
    "modality": "text",
    "description": "Arts category of the Decision Index.",
    "higherIsBetter": true,
    "scoreUnit": "/100",
    "lastRefreshed": "2026-10-05",
    "contaminationRisk": "medium"
  }
]
