{"id":"bench-support-006","slug":"bench-support-006","title":"Policy refund decision — multi-trial pass@3","prompt":"Given order + policy fixtures, approve or deny refund. Graded pass@3 over three trials with identical inputs; DB final state must match gold each trial.","successCriteria":["Correct approve/deny each trial","pass@3 = 1.0","Policy citation in reply"],"scoringHints":["τ-bench-style pass^k","reliabilityConsistency primary"],"agentTypes":["customer_support"],"primaryType":"customer_support","dimensions":["taskSuccess","reliabilityConsistency","instructionFollowing","safetyPolicy"],"difficulty":"medium","timeBudgetSec":700,"tags":["refund","policy","multi-trial","pass@3"],"needsTools":true,"needsWeb":false,"needsCode":false,"holdout":false,"adversarialFlags":[],"type":{"id":"customer_support","slug":"customer_support","name":"Customer-support agent","definition":"Multi-turn agent that resolves user issues using tools/policies (orders, refunds, account state) while staying on-policy, empathetic, and escalation-aware.","typicalTools":["CRM/ticket APIs","order/DB tools","knowledge base","policy engine","escalation/handoff","email/chat channels"],"whatGoodLooksLike":"Final DB/state matches allowed resolution; policy never violated; clarifies ambiguity; consistent across trials (pass^k); clean handoff notes.","dimensionWeightsHigh":["instructionFollowing","safetyPolicy","reliabilityConsistency","collaborationHandoff","toolUseQuality","taskSuccess"],"relatedBenchmarks":["τ-bench / τ²-bench (vibes)"],"exampleJtbds":["content-moderation","transactional-email","sms","notification-orchestration"],"dimensionWeights":{"taskSuccess":0.16,"toolUseQuality":0.1,"planningDecomposition":0.06,"reliabilityConsistency":0.12,"honestyCalibration":0.06,"safetyPolicy":0.12,"efficiency":0.04,"memoryContext":0.04,"instructionFollowing":0.1,"multiStepRobustness":0.04,"collaborationHandoff":0.06,"groundingEvidence":0.02,"communicationClarity":0.06,"adaptivity":0.02},"defaultTaskIds":["bench-support-001","bench-support-002","bench-support-003","bench-support-004","bench-support-005","bench-cross-002","bench-support-006","bench-support-007","bench-support-008"]},"gold":null,"note":"Holdout gold is never returned. Grade via traces + verifiers."}