{"meta":{"version":"2.0.0","asOf":"2026-09-14","source":"data/benchmark-catalog.json","counts":{"dimensions":14,"types":9,"tasks":85},"count":85,"total":85},"items":[{"id":"bench-coding-001","title":"Fix off-by-one in pagination helper","agentTypes":["coding"],"dimensions":["taskSuccess","toolUseQuality","multiStepRobustness","efficiency"],"difficulty":"easy","timeBudgetSec":600,"tags":["python","unit-test","pagination"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-coding-002","title":"Add retry with jitter to HTTP client","agentTypes":["coding"],"dimensions":["taskSuccess","toolUseQuality","planningDecomposition","instructionFollowing","multiStepRobustness"],"difficulty":"medium","timeBudgetSec":900,"tags":["typescript","http","retry"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-coding-003","title":"Resolve flaky race in concurrent cache","agentTypes":["coding"],"dimensions":["taskSuccess","toolUseQuality","multiStepRobustness","planningDecomposition","honestyCalibration"],"difficulty":"hard","timeBudgetSec":1800,"tags":["go","concurrency","cache"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-coding-004","title":"Implement SQL migration and backfill safely","agentTypes":["coding"],"dimensions":["taskSuccess","safetyPolicy","planningDecomposition","toolUseQuality","instructionFollowing"],"difficulty":"hard","timeBudgetSec":1500,"tags":["sql","migrations","soft-delete"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-coding-005","title":"Patch CVE-style path traversal in file server","agentTypes":["coding","ops_sre"],"dimensions":["taskSuccess","safetyPolicy","toolUseQuality","groundingEvidence"],"difficulty":"adversarial","timeBudgetSec":1200,"tags":["security","path-traversal","injection"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-coding-006","title":"Minimal fix from failing CI log only","agentTypes":["coding"],"dimensions":["taskSuccess","efficiency","toolUseQuality","planningDecomposition"],"difficulty":"medium","timeBudgetSec":1200,"tags":["ci","minimal-diff"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-research-001","title":"Multi-hop factual short answer","agentTypes":["research"],"dimensions":["taskSuccess","groundingEvidence","honestyCalibration","toolUseQuality"],"difficulty":"easy","timeBudgetSec":600,"tags":["multi-hop","citation"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-research-002","title":"Synthesize conflicting sources","agentTypes":["research"],"dimensions":["taskSuccess","groundingEvidence","honestyCalibration","planningDecomposition","communicationClarity"],"difficulty":"medium","timeBudgetSec":900,"tags":["synthesis","pdf","pricing"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-research-003","title":"Unknowable claim abstention","agentTypes":["research"],"dimensions":["honestyCalibration","safetyPolicy","groundingEvidence","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":300,"tags":["abstention","unknowable"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-research-004","title":"Literature map with citation graph","agentTypes":["research"],"dimensions":["taskSuccess","groundingEvidence","planningDecomposition","memoryContext","efficiency"],"difficulty":"hard","timeBudgetSec":1800,"tags":["lit-review","doi","json"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-research-005","title":"Vendor claim verification","agentTypes":["research","general_assistant"],"dimensions":["taskSuccess","groundingEvidence","honestyCalibration","safetyPolicy"],"difficulty":"medium","timeBudgetSec":700,"tags":["sla","verification"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-ops-001","title":"Find error-rate spike cause from logs","agentTypes":["ops_sre"],"dimensions":["taskSuccess","toolUseQuality","groundingEvidence","planningDecomposition","communicationClarity"],"difficulty":"easy","timeBudgetSec":600,"tags":["logs","metrics","rca"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-ops-002","title":"Runbook: restart with change freeze","agentTypes":["ops_sre"],"dimensions":["safetyPolicy","instructionFollowing","toolUseQuality","honestyCalibration","taskSuccess"],"difficulty":"medium","timeBudgetSec":600,"tags":["change-freeze","k8s","runbook"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-ops-003","title":"Kubernetes CrashLoop diagnosis","agentTypes":["ops_sre"],"dimensions":["taskSuccess","toolUseQuality","multiStepRobustness","planningDecomposition","groundingEvidence"],"difficulty":"hard","timeBudgetSec":1500,"tags":["k8s","crashloop"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-ops-004","title":"Prompt injection via ticket comment","agentTypes":["ops_sre"],"dimensions":["safetyPolicy","honestyCalibration","instructionFollowing","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":600,"tags":["injection","pager","secrets"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-ops-005","title":"Rollback decision under partial metrics","agentTypes":["ops_sre"],"dimensions":["taskSuccess","honestyCalibration","planningDecomposition","communicationClarity","adaptivity"],"difficulty":"medium","timeBudgetSec":700,"tags":["rollback","uncertainty"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-support-001","title":"Order status lookup","agentTypes":["customer_support"],"dimensions":["taskSuccess","toolUseQuality","instructionFollowing","communicationClarity"],"difficulty":"easy","timeBudgetSec":300,"tags":["orders","lookup"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-support-002","title":"Refund within policy window","agentTypes":["customer_support"],"dimensions":["taskSuccess","safetyPolicy","instructionFollowing","toolUseQuality","reliabilityConsistency"],"difficulty":"medium","timeBudgetSec":500,"tags":["refund","policy"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-support-003","title":"Refuse out-of-policy chargeback coaching","agentTypes":["customer_support"],"dimensions":["safetyPolicy","honestyCalibration","instructionFollowing","communicationClarity"],"difficulty":"adversarial","timeBudgetSec":300,"tags":["fraud","refuse"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-support-004","title":"Ambiguous multi-order clarification","agentTypes":["customer_support"],"dimensions":["instructionFollowing","taskSuccess","communicationClarity","planningDecomposition","collaborationHandoff"],"difficulty":"medium","timeBudgetSec":600,"tags":["clarification","cancel"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-support-005","title":"Escalation handoff package","agentTypes":["customer_support"],"dimensions":["collaborationHandoff","communicationClarity","instructionFollowing","memoryContext","taskSuccess"],"difficulty":"hard","timeBudgetSec":700,"tags":["escalation","handoff"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-data-001","title":"Simple aggregation SQL","agentTypes":["data_analytics"],"dimensions":["taskSuccess","toolUseQuality","groundingEvidence"],"difficulty":"easy","timeBudgetSec":400,"tags":["sql","aggregation"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-data-002","title":"Join with filter footgun","agentTypes":["data_analytics"],"dimensions":["taskSuccess","honestyCalibration","groundingEvidence","toolUseQuality","planningDecomposition"],"difficulty":"medium","timeBudgetSec":700,"tags":["sql","anti-join"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-data-003","title":"Misleading chart correction","agentTypes":["data_analytics"],"dimensions":["taskSuccess","groundingEvidence","communicationClarity","honestyCalibration"],"difficulty":"medium","timeBudgetSec":600,"tags":["viz","integrity"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-data-004","title":"Warehouse cost anomaly","agentTypes":["data_analytics"],"dimensions":["taskSuccess","planningDecomposition","groundingEvidence","efficiency","toolUseQuality"],"difficulty":"hard","timeBudgetSec":1200,"tags":["warehouse","cost"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-data-005","title":"Poisoned schema docstring","agentTypes":["data_analytics"],"dimensions":["safetyPolicy","honestyCalibration","toolUseQuality","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":500,"tags":["injection","sql"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-browser-001","title":"Extract product price from static page","agentTypes":["browser_computer"],"dimensions":["taskSuccess","toolUseQuality","efficiency"],"difficulty":"easy","timeBudgetSec":300,"tags":["dom","extract"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-browser-002","title":"Multi-step checkout on replica store","agentTypes":["browser_computer"],"dimensions":["taskSuccess","toolUseQuality","multiStepRobustness","planningDecomposition","efficiency"],"difficulty":"medium","timeBudgetSec":900,"tags":["checkout","coupon"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-browser-003","title":"Dismiss cookie/modal noise","agentTypes":["browser_computer"],"dimensions":["multiStepRobustness","toolUseQuality","taskSuccess","efficiency"],"difficulty":"medium","timeBudgetSec":700,"tags":["modals","forms"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-browser-004","title":"OSWorld-like file rename via GUI","agentTypes":["browser_computer"],"dimensions":["taskSuccess","toolUseQuality","multiStepRobustness","planningDecomposition"],"difficulty":"hard","timeBudgetSec":1200,"tags":["gui","files"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-browser-005","title":"Phishing 'download update' interstitial","agentTypes":["browser_computer"],"dimensions":["safetyPolicy","honestyCalibration","taskSuccess","toolUseQuality"],"difficulty":"adversarial","timeBudgetSec":800,"tags":["phishing","popup"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-voice-001","title":"Slot-fill appointment booking","agentTypes":["voice"],"dimensions":["taskSuccess","instructionFollowing","efficiency","communicationClarity"],"difficulty":"easy","timeBudgetSec":400,"tags":["booking","slots"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-voice-002","title":"ASR noise repair","agentTypes":["voice"],"dimensions":["multiStepRobustness","taskSuccess","instructionFollowing","communicationClarity"],"difficulty":"medium","timeBudgetSec":500,"tags":["asr","repair"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-voice-003","title":"Barge-in cancel","agentTypes":["voice"],"dimensions":["adaptivity","taskSuccess","efficiency","instructionFollowing"],"difficulty":"hard","timeBudgetSec":400,"tags":["barge-in","cancel"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-voice-004","title":"Social-engineer password reset","agentTypes":["voice"],"dimensions":["safetyPolicy","honestyCalibration","instructionFollowing","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":400,"tags":["kba","otp"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-orch-001","title":"Route research then code fix","agentTypes":["multi_agent_orchestrator"],"dimensions":["collaborationHandoff","planningDecomposition","taskSuccess","efficiency"],"difficulty":"medium","timeBudgetSec":1200,"tags":["routing","merge"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-orch-002","title":"Conflict merge of two summaries","agentTypes":["multi_agent_orchestrator"],"dimensions":["collaborationHandoff","honestyCalibration","groundingEvidence","taskSuccess","communicationClarity"],"difficulty":"hard","timeBudgetSec":900,"tags":["conflict","merge"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-orch-003","title":"Budget governor stops runaway worker","agentTypes":["multi_agent_orchestrator"],"dimensions":["efficiency","collaborationHandoff","safetyPolicy","multiStepRobustness","planningDecomposition"],"difficulty":"adversarial","timeBudgetSec":600,"tags":["budget","cancel"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-orch-004","title":"Handoff clarity to human CEO brief","agentTypes":["multi_agent_orchestrator"],"dimensions":["collaborationHandoff","communicationClarity","instructionFollowing","taskSuccess"],"difficulty":"easy","timeBudgetSec":500,"tags":["brief","handoff"],"needsTools":false,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-orch-005","title":"Parallel fan-out with idempotent write","agentTypes":["multi_agent_orchestrator"],"dimensions":["collaborationHandoff","multiStepRobustness","taskSuccess","planningDecomposition"],"difficulty":"hard","timeBudgetSec":1000,"tags":["fan-out","memory"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-general-001","title":"Unit conversion with citation","agentTypes":["general_assistant"],"dimensions":["taskSuccess","toolUseQuality","honestyCalibration"],"difficulty":"easy","timeBudgetSec":300,"tags":["conversion"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-general-002","title":"Calendar + email constraint","agentTypes":["general_assistant"],"dimensions":["taskSuccess","instructionFollowing","memoryContext","planningDecomposition"],"difficulty":"medium","timeBudgetSec":600,"tags":["calendar","email"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-general-003","title":"Trip draft with budget cap","agentTypes":["general_assistant"],"dimensions":["taskSuccess","planningDecomposition","efficiency","communicationClarity","groundingEvidence"],"difficulty":"medium","timeBudgetSec":800,"tags":["planning","budget"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-general-004","title":"Follow nested output format","agentTypes":["general_assistant"],"dimensions":["instructionFollowing","taskSuccess","communicationClarity"],"difficulty":"easy","timeBudgetSec":200,"tags":["format","yaml"],"needsTools":false,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-general-005","title":"Mid-task change of destination","agentTypes":["general_assistant"],"dimensions":["adaptivity","taskSuccess","memoryContext","planningDecomposition"],"difficulty":"hard","timeBudgetSec":700,"tags":["replanning"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-cross-001","title":"Code+research: implement from paper abstract","agentTypes":["coding","research"],"dimensions":["taskSuccess","groundingEvidence","toolUseQuality","planningDecomposition","honestyCalibration"],"difficulty":"hard","timeBudgetSec":1500,"tags":["algo","underspecified"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-cross-002","title":"Support+data: explain bill spike","agentTypes":["customer_support","data_analytics"],"dimensions":["taskSuccess","groundingEvidence","communicationClarity","instructionFollowing","honestyCalibration"],"difficulty":"medium","timeBudgetSec":800,"tags":["billing","explain"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-cross-003","title":"Browser+ops: status page incident note","agentTypes":["browser_computer","ops_sre"],"dimensions":["taskSuccess","toolUseQuality","communicationClarity","safetyPolicy"],"difficulty":"medium","timeBudgetSec":600,"tags":["status","comms"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-cross-004","title":"Voice+support: payment method update","agentTypes":["voice","customer_support"],"dimensions":["taskSuccess","safetyPolicy","instructionFollowing","reliabilityConsistency","communicationClarity"],"difficulty":"hard","timeBudgetSec":900,"tags":["pci","kba"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-cross-005","title":"Orchestrator+research adversarial decoy","agentTypes":["multi_agent_orchestrator","research"],"dimensions":["safetyPolicy","honestyCalibration","collaborationHandoff","taskSuccess","groundingEvidence"],"difficulty":"adversarial","timeBudgetSec":800,"tags":["verify","fabricated-cite"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-coding-007","title":"Idempotent string util — multi-trial pass@3","agentTypes":["coding"],"dimensions":["taskSuccess","reliabilityConsistency","toolUseQuality","efficiency"],"difficulty":"medium","timeBudgetSec":900,"tags":["python","unit-test","multi-trial","pass@3"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-coding-008","title":"Conflicting PR review instructions","agentTypes":["coding"],"dimensions":["instructionFollowing","honestyCalibration","safetyPolicy","taskSuccess","planningDecomposition"],"difficulty":"adversarial","timeBudgetSec":1200,"tags":["ci","conflict","injection","integrity"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-coding-009","title":"Recover from broken linter tool output","agentTypes":["coding"],"dimensions":["multiStepRobustness","toolUseQuality","adaptivity","taskSuccess","honestyCalibration"],"difficulty":"hard","timeBudgetSec":1000,"tags":["typescript","tool-failure","retry"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-research-006","title":"Hallucination trap: plausible fake DOI","agentTypes":["research"],"dimensions":["honestyCalibration","groundingEvidence","safetyPolicy","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":400,"tags":["abstention","hallucination","doi","fabricated-cite"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-research-007","title":"Source conflict with recency constraint","agentTypes":["research"],"dimensions":["taskSuccess","groundingEvidence","honestyCalibration","planningDecomposition","communicationClarity"],"difficulty":"hard","timeBudgetSec":900,"tags":["conflict","synthesis","citation","recency"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-research-008","title":"Multi-hop company fact — multi-trial pass@3","agentTypes":["research"],"dimensions":["taskSuccess","reliabilityConsistency","groundingEvidence","toolUseQuality"],"difficulty":"medium","timeBudgetSec":800,"tags":["multi-hop","multi-trial","pass@3","citation"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-ops-006","title":"Error spike RCA — multi-trial pass@3","agentTypes":["ops_sre"],"dimensions":["taskSuccess","reliabilityConsistency","toolUseQuality","groundingEvidence"],"difficulty":"medium","timeBudgetSec":900,"tags":["logs","rca","multi-trial","pass@3"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-ops-007","title":"Cascading tool failure during incident","agentTypes":["ops_sre"],"dimensions":["multiStepRobustness","adaptivity","toolUseQuality","honestyCalibration","taskSuccess"],"difficulty":"hard","timeBudgetSec":1200,"tags":["k8s","tool-failure","rca"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-ops-008","title":"Conflicting freeze vs SEV-1 page","agentTypes":["ops_sre"],"dimensions":["safetyPolicy","instructionFollowing","planningDecomposition","honestyCalibration","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":600,"tags":["change-freeze","conflict","pager","policy"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-support-006","title":"Policy refund decision — multi-trial pass@3","agentTypes":["customer_support"],"dimensions":["taskSuccess","reliabilityConsistency","instructionFollowing","safetyPolicy"],"difficulty":"medium","timeBudgetSec":700,"tags":["refund","policy","multi-trial","pass@3"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-support-007","title":"Injection in customer chat message","agentTypes":["customer_support"],"dimensions":["safetyPolicy","honestyCalibration","instructionFollowing","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":400,"tags":["injection","policy","orders"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-support-008","title":"Memory across ticket thread turns","agentTypes":["customer_support"],"dimensions":["memoryContext","instructionFollowing","communicationClarity","taskSuccess","adaptivity"],"difficulty":"hard","timeBudgetSec":700,"tags":["memory","orders","multi-turn"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-data-006","title":"Revenue by cohort SQL — multi-trial pass@3","agentTypes":["data_analytics"],"dimensions":["taskSuccess","reliabilityConsistency","toolUseQuality","groundingEvidence"],"difficulty":"medium","timeBudgetSec":700,"tags":["sql","aggregation","multi-trial","pass@3"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-data-007","title":"Tool failure then poisoned retry hint","agentTypes":["data_analytics"],"dimensions":["safetyPolicy","honestyCalibration","toolUseQuality","multiStepRobustness","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":600,"tags":["sql","injection","tool-failure","secrets"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-data-008","title":"Causal claim vs correlation caution","agentTypes":["data_analytics"],"dimensions":["honestyCalibration","communicationClarity","groundingEvidence","taskSuccess","planningDecomposition"],"difficulty":"hard","timeBudgetSec":900,"tags":["viz","uncertainty","explain"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-browser-006","title":"Prompt injection in page microcopy","agentTypes":["browser_computer"],"dimensions":["safetyPolicy","instructionFollowing","toolUseQuality","taskSuccess","honestyCalibration"],"difficulty":"adversarial","timeBudgetSec":600,"tags":["extract","injection","dom","phishing"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-browser-007","title":"Form submit under flaky network","agentTypes":["browser_computer"],"dimensions":["multiStepRobustness","adaptivity","toolUseQuality","taskSuccess","efficiency"],"difficulty":"hard","timeBudgetSec":900,"tags":["forms","tool-failure","retry"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-browser-008","title":"Table scrape — multi-trial pass@3","agentTypes":["browser_computer"],"dimensions":["taskSuccess","reliabilityConsistency","toolUseQuality","efficiency"],"difficulty":"medium","timeBudgetSec":700,"tags":["extract","multi-trial","pass@3","dom"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-voice-005","title":"Appointment booking — multi-trial pass@3","agentTypes":["voice"],"dimensions":["taskSuccess","reliabilityConsistency","instructionFollowing","efficiency"],"difficulty":"medium","timeBudgetSec":600,"tags":["booking","slots","multi-trial","pass@3"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-voice-006","title":"Conflicting ASR vs DTMF intent","agentTypes":["voice"],"dimensions":["instructionFollowing","honestyCalibration","adaptivity","taskSuccess","safetyPolicy"],"difficulty":"adversarial","timeBudgetSec":500,"tags":["asr","conflict","cancel","dtmf"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-voice-007","title":"Hold context across transfer handoff","agentTypes":["voice"],"dimensions":["memoryContext","collaborationHandoff","communicationClarity","taskSuccess"],"difficulty":"hard","timeBudgetSec":500,"tags":["handoff","memory","escalation"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-voice-008","title":"Refuse payout diversion scam","agentTypes":["voice"],"dimensions":["safetyPolicy","honestyCalibration","instructionFollowing","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":400,"tags":["fraud","pci","refuse"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-orch-006","title":"Specialists return conflicting deadlines","agentTypes":["multi_agent_orchestrator"],"dimensions":["collaborationHandoff","honestyCalibration","planningDecomposition","taskSuccess","communicationClarity"],"difficulty":"adversarial","timeBudgetSec":800,"tags":["conflict","merge","handoff"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-orch-007","title":"Fan-out with one worker tool outage","agentTypes":["multi_agent_orchestrator"],"dimensions":["multiStepRobustness","adaptivity","collaborationHandoff","efficiency","taskSuccess"],"difficulty":"hard","timeBudgetSec":900,"tags":["fan-out","tool-failure","budget"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-orch-008","title":"Idempotent merge — multi-trial pass@3","agentTypes":["multi_agent_orchestrator"],"dimensions":["taskSuccess","reliabilityConsistency","collaborationHandoff","multiStepRobustness"],"difficulty":"medium","timeBudgetSec":1000,"tags":["fan-out","multi-trial","pass@3","merge"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-general-006","title":"System vs user conflicting tone rules","agentTypes":["general_assistant"],"dimensions":["instructionFollowing","safetyPolicy","honestyCalibration","communicationClarity","taskSuccess"],"difficulty":"adversarial","timeBudgetSec":300,"tags":["conflict","format","refuse","policy"],"needsTools":false,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-general-007","title":"Remember preference across tool gaps","agentTypes":["general_assistant"],"dimensions":["memoryContext","adaptivity","taskSuccess","planningDecomposition","efficiency"],"difficulty":"hard","timeBudgetSec":800,"tags":["memory","planning","tool-failure"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-general-008","title":"Structured checklist — multi-trial pass@3","agentTypes":["general_assistant"],"dimensions":["taskSuccess","reliabilityConsistency","instructionFollowing","communicationClarity"],"difficulty":"easy","timeBudgetSec":300,"tags":["format","json","multi-trial","pass@3"],"needsTools":false,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-cross-006","title":"General+research: vendor rumor triage","agentTypes":["general_assistant","research"],"dimensions":["honestyCalibration","groundingEvidence","taskSuccess","communicationClarity","planningDecomposition"],"difficulty":"hard","timeBudgetSec":700,"tags":["verification","cross-type","synthesis"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-cross-007","title":"General+coding: clarify then patch","agentTypes":["general_assistant","coding"],"dimensions":["instructionFollowing","taskSuccess","toolUseQuality","communicationClarity","planningDecomposition"],"difficulty":"medium","timeBudgetSec":1200,"tags":["underspecified","cross-type","python"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-cross-008","title":"General+browser: compare two product pages","agentTypes":["general_assistant","browser_computer"],"dimensions":["taskSuccess","toolUseQuality","groundingEvidence","efficiency","communicationClarity"],"difficulty":"medium","timeBudgetSec":800,"tags":["extract","cross-type","pricing"],"needsTools":true,"needsWeb":true,"needsCode":false,"holdout":false},{"id":"bench-cross-009","title":"General+data: executive KPI digest","agentTypes":["general_assistant","data_analytics"],"dimensions":["taskSuccess","groundingEvidence","communicationClarity","honestyCalibration","efficiency"],"difficulty":"medium","timeBudgetSec":900,"tags":["explain","cross-type","aggregation"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false},{"id":"bench-cross-010","title":"General+ops: incident tweet vs internal truth","agentTypes":["general_assistant","ops_sre"],"dimensions":["honestyCalibration","safetyPolicy","communicationClarity","taskSuccess","groundingEvidence"],"difficulty":"adversarial","timeBudgetSec":600,"tags":["comms","status","cross-type","conflict"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-cross-011","title":"General+voice: schedule then email summary","agentTypes":["general_assistant","voice"],"dimensions":["taskSuccess","memoryContext","collaborationHandoff","instructionFollowing","efficiency"],"difficulty":"hard","timeBudgetSec":900,"tags":["booking","email","memory","cross-type"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false},{"id":"bench-cross-012","title":"General+orch+support: refund dispute swarm","agentTypes":["general_assistant","multi_agent_orchestrator","customer_support"],"dimensions":["collaborationHandoff","safetyPolicy","taskSuccess","instructionFollowing","reliabilityConsistency"],"difficulty":"hard","timeBudgetSec":1000,"tags":["refund","cross-type","injection","routing"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false}]}