{"id":"bench-data-006","slug":"bench-data-006","title":"Revenue by cohort SQL — multi-trial pass@3","prompt":"Write SQL returning 2025 Q1 revenue by signup cohort. Fixture DB; grade CSV match. pass@3: three independent query submissions must all match gold.","successCriteria":["CSV matches gold each trial","pass@3 = 1.0","No mutating statements"],"scoringHints":["Deterministic grader","reliabilityConsistency from pass@3"],"agentTypes":["data_analytics"],"primaryType":"data_analytics","dimensions":["taskSuccess","reliabilityConsistency","toolUseQuality","groundingEvidence"],"difficulty":"medium","timeBudgetSec":700,"tags":["sql","aggregation","multi-trial","pass@3"],"needsTools":true,"needsWeb":false,"needsCode":true,"holdout":false,"adversarialFlags":[],"type":{"id":"data_analytics","slug":"data_analytics","name":"Data / analytics agent","definition":"Agent that explores datasets, writes correct SQL/pandas/Spark, validates assumptions, and produces decision-ready analyses with reproducible steps.","typicalTools":["SQL engines","dataframe libs","plotting","schema catalog","notebook/REPL","warehouse APIs"],"whatGoodLooksLike":"Correct numbers vs gold queries; schema-aware; documents filters/joins; flags data quality issues; no silent row drops; reproducible query artifacts.","dimensionWeightsHigh":["taskSuccess","groundingEvidence","toolUseQuality","honestyCalibration","planningDecomposition","efficiency"],"relatedBenchmarks":["AgentBench DB (vibes)","text-to-SQL suites (vibes)"],"exampleJtbds":["spreadsheet-sql-agents","sql-warehouse-agent-context","lakehouse-llm","agent-analytics-warehouse"],"dimensionWeights":{"taskSuccess":0.2,"toolUseQuality":0.1,"planningDecomposition":0.08,"reliabilityConsistency":0.06,"honestyCalibration":0.1,"safetyPolicy":0.04,"efficiency":0.08,"memoryContext":0.04,"instructionFollowing":0.06,"multiStepRobustness":0.06,"collaborationHandoff":0.02,"groundingEvidence":0.1,"communicationClarity":0.04,"adaptivity":0.02},"defaultTaskIds":["bench-data-001","bench-data-002","bench-data-003","bench-data-004","bench-data-005","bench-data-006","bench-data-007","bench-data-008"]},"gold":null,"note":"Holdout gold is never returned. Grade via traces + verifiers."}