{"meta":{"version":"2.0.0","asOf":"2026-09-14","source":"data/benchmark-catalog.json","counts":{"dimensions":14,"types":9,"tasks":85},"count":9},"items":[{"id":"coding","slug":"coding","name":"Coding agent","definition":"Autonomous or semi-autonomous agent that reads/writes code, runs tests, navigates repos, and lands patches or PRs against executable success criteria.","typicalTools":["filesystem","shell/terminal","git","linters/test runners","language servers","package managers","diff/patch tools","CI APIs"],"whatGoodLooksLike":"Correct patches that pass hidden tests; minimal unrelated churn; recovers from failing tests; respects repo conventions; cites file:line evidence; does not invent APIs.","dimensionWeightsHigh":["taskSuccess","toolUseQuality","multiStepRobustness","instructionFollowing","efficiency","honestyCalibration"],"relatedBenchmarks":["SWE-bench Verified (vibes)","Terminal-Bench (vibes)"],"exampleJtbds":["coding-agent","diff-aware-pr-review","agent-sandbox","code-interpreter"],"dimensionWeights":{"taskSuccess":0.2,"toolUseQuality":0.14,"planningDecomposition":0.08,"reliabilityConsistency":0.08,"honestyCalibration":0.08,"safetyPolicy":0.06,"efficiency":0.1,"memoryContext":0.03,"instructionFollowing":0.08,"multiStepRobustness":0.08,"collaborationHandoff":0.02,"groundingEvidence":0.03,"communicationClarity":0.01,"adaptivity":0.01},"defaultTaskIds":["bench-coding-001","bench-coding-002","bench-coding-003","bench-coding-004","bench-coding-005","bench-coding-006","bench-cross-001","bench-coding-007","bench-coding-008","bench-coding-009"],"coverage":{"mentions":11,"gate":7}},{"id":"research","slug":"research","name":"Research agent","definition":"Agent that gathers, synthesizes, and cites evidence from web/docs/corpora to answer open-ended or multi-hop questions with traceable grounding.","typicalTools":["web search","browser fetch","PDF/OCR","citation managers","note/memory stores","academic APIs","scrapers (policy-compliant)"],"whatGoodLooksLike":"Answers match ground truth or rubric; every non-trivial claim has a source; distinguishes unknown vs known; no fabricated citations; clear uncertainty.","dimensionWeightsHigh":["taskSuccess","groundingEvidence","honestyCalibration","planningDecomposition","memoryContext","safetyPolicy"],"relatedBenchmarks":["GAIA / Gaia2 (vibes)","BrowseComp-style retrieval"],"exampleJtbds":["deep-research-agents","agent-search","web-grounded-qa","search-augmented-llm"],"dimensionWeights":{"taskSuccess":0.18,"toolUseQuality":0.08,"planningDecomposition":0.1,"reliabilityConsistency":0.06,"honestyCalibration":0.12,"safetyPolicy":0.08,"efficiency":0.04,"memoryContext":0.06,"instructionFollowing":0.04,"multiStepRobustness":0.04,"collaborationHandoff":0.02,"groundingEvidence":0.1,"communicationClarity":0.04,"adaptivity":0.04},"defaultTaskIds":["bench-research-001","bench-research-002","bench-research-003","bench-research-004","bench-research-005","bench-research-006","bench-research-007","bench-research-008"],"coverage":{"mentions":11,"gate":7}},{"id":"ops_sre","slug":"ops_sre","name":"Ops / SRE agent","definition":"Agent that diagnoses incidents, inspects logs/metrics/traces, proposes or applies runbook actions, and communicates blast-radius-aware remediations under change-control constraints.","typicalTools":["log query","metrics/APM","k8s/cloud CLIs","runbook store","paging/ticketing","feature flags","shell (scoped)"],"whatGoodLooksLike":"Correct root-cause hypotheses with evidence; safe action ordering; respects freeze/change windows; clear rollback; no destructive commands without confirmation gates.","dimensionWeightsHigh":["safetyPolicy","toolUseQuality","multiStepRobustness","honestyCalibration","planningDecomposition","instructionFollowing"],"relatedBenchmarks":["AgentBench OS/DB (vibes)","IT automation / Terminal-Bench (vibes)"],"exampleJtbds":["agent-error-observability","infra-observability","llm-tracing","cost-monitoring"],"dimensionWeights":{"taskSuccess":0.14,"toolUseQuality":0.1,"planningDecomposition":0.08,"reliabilityConsistency":0.1,"honestyCalibration":0.08,"safetyPolicy":0.14,"efficiency":0.06,"memoryContext":0.03,"instructionFollowing":0.08,"multiStepRobustness":0.08,"collaborationHandoff":0.04,"groundingEvidence":0.03,"communicationClarity":0.02,"adaptivity":0.02},"defaultTaskIds":["bench-ops-001","bench-ops-002","bench-ops-003","bench-ops-004","bench-ops-005","bench-ops-006","bench-ops-007","bench-ops-008"],"coverage":{"mentions":11,"gate":7}},{"id":"customer_support","slug":"customer_support","name":"Customer-support agent","definition":"Multi-turn agent that resolves user issues using tools/policies (orders, refunds, account state) while staying on-policy, empathetic, and escalation-aware.","typicalTools":["CRM/ticket APIs","order/DB tools","knowledge base","policy engine","escalation/handoff","email/chat channels"],"whatGoodLooksLike":"Final DB/state matches allowed resolution; policy never violated; clarifies ambiguity; consistent across trials (pass^k); clean handoff notes.","dimensionWeightsHigh":["instructionFollowing","safetyPolicy","reliabilityConsistency","collaborationHandoff","toolUseQuality","taskSuccess"],"relatedBenchmarks":["τ-bench / τ²-bench (vibes)"],"exampleJtbds":["content-moderation","transactional-email","sms","notification-orchestration"],"dimensionWeights":{"taskSuccess":0.16,"toolUseQuality":0.1,"planningDecomposition":0.06,"reliabilityConsistency":0.12,"honestyCalibration":0.06,"safetyPolicy":0.12,"efficiency":0.04,"memoryContext":0.04,"instructionFollowing":0.1,"multiStepRobustness":0.04,"collaborationHandoff":0.06,"groundingEvidence":0.02,"communicationClarity":0.06,"adaptivity":0.02},"defaultTaskIds":["bench-support-001","bench-support-002","bench-support-003","bench-support-004","bench-support-005","bench-cross-002","bench-support-006","bench-support-007","bench-support-008"],"coverage":{"mentions":11,"gate":7}},{"id":"data_analytics","slug":"data_analytics","name":"Data / analytics agent","definition":"Agent that explores datasets, writes correct SQL/pandas/Spark, validates assumptions, and produces decision-ready analyses with reproducible steps.","typicalTools":["SQL engines","dataframe libs","plotting","schema catalog","notebook/REPL","warehouse APIs"],"whatGoodLooksLike":"Correct numbers vs gold queries; schema-aware; documents filters/joins; flags data quality issues; no silent row drops; reproducible query artifacts.","dimensionWeightsHigh":["taskSuccess","groundingEvidence","toolUseQuality","honestyCalibration","planningDecomposition","efficiency"],"relatedBenchmarks":["AgentBench DB (vibes)","text-to-SQL suites (vibes)"],"exampleJtbds":["spreadsheet-sql-agents","sql-warehouse-agent-context","lakehouse-llm","agent-analytics-warehouse"],"dimensionWeights":{"taskSuccess":0.2,"toolUseQuality":0.1,"planningDecomposition":0.08,"reliabilityConsistency":0.06,"honestyCalibration":0.1,"safetyPolicy":0.04,"efficiency":0.08,"memoryContext":0.04,"instructionFollowing":0.06,"multiStepRobustness":0.06,"collaborationHandoff":0.02,"groundingEvidence":0.1,"communicationClarity":0.04,"adaptivity":0.02},"defaultTaskIds":["bench-data-001","bench-data-002","bench-data-003","bench-data-004","bench-data-005","bench-data-006","bench-data-007","bench-data-008"],"coverage":{"mentions":10,"gate":6}},{"id":"browser_computer","slug":"browser_computer","name":"Browser / computer-use agent","definition":"Agent that operates GUIs or browsers (click, type, navigate, upload) to complete end-to-end workflows in web or desktop environments.","typicalTools":["browser automation","screenshot/a11y tree","mouse/keyboard","download/upload","DOM/selector tools","OS desktop control"],"whatGoodLooksLike":"Task end-state verified by execution scripts; stable selectors/grounding; recovers from popups/noise; avoids loops; respects site ToS in live mode.","dimensionWeightsHigh":["taskSuccess","toolUseQuality","multiStepRobustness","efficiency","safetyPolicy","planningDecomposition"],"relatedBenchmarks":["WebArena (vibes)","OSWorld (vibes)","Mind2Web (vibes)"],"exampleJtbds":["web-agent","computer-use-browser","cloud-browser","browser-agent-loop"],"dimensionWeights":{"taskSuccess":0.2,"toolUseQuality":0.14,"planningDecomposition":0.08,"reliabilityConsistency":0.08,"honestyCalibration":0.04,"safetyPolicy":0.08,"efficiency":0.1,"memoryContext":0.02,"instructionFollowing":0.06,"multiStepRobustness":0.12,"collaborationHandoff":0.02,"groundingEvidence":0.02,"communicationClarity":0.02,"adaptivity":0.02},"defaultTaskIds":["bench-browser-001","bench-browser-002","bench-browser-003","bench-browser-004","bench-browser-005","bench-cross-003","bench-browser-006","bench-browser-007","bench-browser-008"],"coverage":{"mentions":10,"gate":6}},{"id":"voice","slug":"voice","name":"Voice agent","definition":"Real-time or turn-based spoken agent handling ASR noise, barge-in, latency budgets, and spoken dialogue policy for booking, support, or IVR-like flows.","typicalTools":["ASR/TTS","telephony/WebRTC","turn-taking/VAD","CRM tools","calendar","interrupt handlers"],"whatGoodLooksLike":"Goal completed within latency SLO; graceful repair on ASR errors; confirms critical slots; never invents confirmations; clean escalation to human.","dimensionWeightsHigh":["taskSuccess","efficiency","reliabilityConsistency","instructionFollowing","safetyPolicy","multiStepRobustness"],"relatedBenchmarks":["Spoken dialogue / voice-bench vibes; latency-first custom evals"],"exampleJtbds":["voice","voice-agents","voice-agent-platform","speech-to-text"],"dimensionWeights":{"taskSuccess":0.16,"toolUseQuality":0.06,"planningDecomposition":0.05,"reliabilityConsistency":0.14,"honestyCalibration":0.05,"safetyPolicy":0.12,"efficiency":0.14,"memoryContext":0.03,"instructionFollowing":0.1,"multiStepRobustness":0.06,"collaborationHandoff":0.03,"groundingEvidence":0.01,"communicationClarity":0.04,"adaptivity":0.01},"defaultTaskIds":["bench-voice-001","bench-voice-002","bench-voice-003","bench-voice-004","bench-cross-004","bench-voice-005","bench-voice-006","bench-voice-007","bench-voice-008"],"coverage":{"mentions":10,"gate":6}},{"id":"multi_agent_orchestrator","slug":"multi_agent_orchestrator","name":"Multi-agent orchestrator","definition":"Controller that decomposes work, routes to specialist agents/tools, merges results, resolves conflicts, and produces a coherent final artifact with clear ownership.","typicalTools":["agent registry","message bus / A2A","shared memory","router/planner","critic/verifier agents","budget governor"],"whatGoodLooksLike":"Correct merge without duplicated work; explicit handoffs; conflict detection; budget adherence; failure isolation; auditable assignment log.","dimensionWeightsHigh":["collaborationHandoff","planningDecomposition","taskSuccess","efficiency","multiStepRobustness","honestyCalibration"],"relatedBenchmarks":["Multi-agent coordination evals; ARE/Gaia2 multi-actor vibes"],"exampleJtbds":["durable-agent-workflows","langgraph-deploy","agent-tool-router","stateful-agent-runtime"],"dimensionWeights":{"taskSuccess":0.15,"toolUseQuality":0.06,"planningDecomposition":0.14,"reliabilityConsistency":0.05,"honestyCalibration":0.06,"safetyPolicy":0.05,"efficiency":0.08,"memoryContext":0.05,"instructionFollowing":0.05,"multiStepRobustness":0.08,"collaborationHandoff":0.15,"groundingEvidence":0.03,"communicationClarity":0.03,"adaptivity":0.02},"defaultTaskIds":["bench-orch-001","bench-orch-002","bench-orch-003","bench-orch-004","bench-orch-005","bench-cross-005","bench-orch-006","bench-orch-007","bench-orch-008"],"coverage":{"mentions":10,"gate":6}},{"id":"general_assistant","slug":"general_assistant","name":"General assistant","definition":"Broad personal/work assistant spanning scheduling, light research, writing, and tool use without deep specialization—GAIA-style real-world questions.","typicalTools":["web search","calendar/email","files","calculator","browser","memory"],"whatGoodLooksLike":"High success on short-answer verifiable tasks; good tool selection under ambiguity; honest when stuck; efficient path length.","dimensionWeightsHigh":["taskSuccess","toolUseQuality","honestyCalibration","planningDecomposition","instructionFollowing","efficiency"],"relatedBenchmarks":["GAIA / Gaia2 (vibes)"],"exampleJtbds":["frontier-chat","long-context-agents","multimodal","cross-session-user-memory"],"dimensionWeights":{"taskSuccess":0.18,"toolUseQuality":0.1,"planningDecomposition":0.1,"reliabilityConsistency":0.06,"honestyCalibration":0.12,"safetyPolicy":0.08,"efficiency":0.08,"memoryContext":0.05,"instructionFollowing":0.08,"multiStepRobustness":0.04,"collaborationHandoff":0.02,"groundingEvidence":0.05,"communicationClarity":0.03,"adaptivity":0.01},"defaultTaskIds":["bench-general-001","bench-general-002","bench-general-003","bench-general-004","bench-general-005","bench-general-006","bench-general-007","bench-general-008","bench-cross-006","bench-cross-007","bench-cross-008","bench-cross-009","bench-cross-010","bench-cross-011","bench-cross-012"],"coverage":{"mentions":16,"gate":10}}]}