{"jtbd":{"id":"public-agent-benchmarks","slug":"public-agent-benchmarks","name":"Public Agent Capability Benchmarks (SWE-bench, τ-bench, GAIA, OSWorld, WebArena, Terminal-Bench, BFCL, MLE-bench)","shortName":"Public agent capability benchmarks"},"recommended":{"role":"depot_pick","id":"promptfoo","slug":"promptfoo","name":"Promptfoo","website":"https://www.promptfoo.dev","editorialOverall":8.44,"scoreKind":"curated_estimate","scoreAsOf":"2026-10-28","price":{"available":false,"unavailableReason":"no_sourced_price","skus":[],"normalizedUsdPerMillionTokens":null,"normalizationStatus":"unavailable","normalizationReason":"no_sourced_price"}},"priceComparison":{"skuKind":"none","unit":null,"comparison":"no comparable sourced unit for this job; chat input $/1M and other units were not borrowed","amountLabel":"Matching price","reportsUsdPerMillion":false,"matchedCandidates":0},"cheapestInput":null,"cheapest":null,"recommendedWorkloadQuote":null,"representativeQuote":null,"estimatedTaskCost":null,"cheapestGoodEnough":{"band":{"rule":"editorial overall >= Depot Pick overall - 0.5","minOverall":7.94,"pickOverall":8.44,"points":0.5},"pick":null,"reason":"no_matching_sourced_price","cheaperButBelowBand":[]},"dates":{"scoreAsOf":"2026-10-28","scoreKind":"curated_estimate","priceVerifiedAsOf":null,"note":"scoreAsOf is when the editorial score was last set; priceVerifiedAsOf is the newest fact-check of the pick's price rows. They are separate: a fresh price check does not re-score."},"sourcedInputs":[],"pickIsPriceWinner":null,"priceWinnerReason":"no_comparison_no_candidate_has_matching_sourced_price","priceWinnerNote":"Nothing was compared. no comparable sourced unit for this job; chat input $/1M and other units were not borrowed pickIsPriceWinner is null.","excluded":3,"coverage":{"candidates":5,"withAnySourcedRow":2,"withSourcedInputUsdPerMillion":0,"withMatchingPrice":0,"excludedNoSourcedPrice":3,"excludedFromCheapestInput":5},"normalizedUsdPerMillionTokens":null,"normalizationReason":"input_and_output_are_not_blended","seo":{"published":false,"reason":"Only 0 candidate(s) have a matching Matching price (none). Need 3 before a crawlable price section. Quote JSON is still returned."},"skipTopPick":false,"matchedConstraints":[],"unsupportedConstraints":[],"routePlan":{"jtbdId":"public-agent-benchmarks","primary":{"providerId":"promptfoo","name":"Promptfoo","overall":8.44,"scoreBand":{"low":7.9,"mid":8.4,"high":8.9,"halfWidth":0.5,"basis":"editorial_prior","display":"7.9–8.9 (editorial prior)"},"scoreLayers":{"editorialPrior":8.44,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"primary","reason":"Editorial Depot Pick for Public Agent Capability Benchmarks (SWE-bench, τ-bench, GAIA, OSWorld, WebArena, Terminal-Bench, BFCL, MLE-bench); healthy/eligible under caller constraints."},"omittedReason":null,"fallback":[{"providerId":"braintrust","name":"Braintrust","overall":8.22,"scoreBand":{"low":7.6,"mid":8.2,"high":8.8,"halfWidth":0.6,"basis":"editorial_prior","display":"7.6–8.8 (editorial prior)"},"scoreLayers":{"editorialPrior":8.22,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Public Agent Capability Benchmarks (SWE-bench, τ-bench, GAIA, OSWorld, WebArena, Terminal-Bench, BFCL, MLE-bench)."},{"providerId":"e2b","name":"E2B","overall":8.59,"scoreBand":{"low":8,"mid":8.6,"high":9.2,"halfWidth":0.6,"basis":"editorial_prior","display":"8.0–9.2 (editorial prior)"},"scoreLayers":{"editorialPrior":8.59,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Public Agent Capability Benchmarks (SWE-bench, τ-bench, GAIA, OSWorld, WebArena, Terminal-Bench, BFCL, MLE-bench)."},{"providerId":"modal","name":"Modal","overall":8.46,"scoreBand":{"low":7.9,"mid":8.5,"high":9.1,"halfWidth":0.6,"basis":"editorial_prior","display":"7.9–9.1 (editorial prior)"},"scoreLayers":{"editorialPrior":8.46,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Public Agent Capability Benchmarks (SWE-bench, τ-bench, GAIA, OSWorld, WebArena, Terminal-Bench, BFCL, MLE-bench)."},{"providerId":"weights-biases","name":"Weights & Biases","overall":7.95,"scoreBand":{"low":7.4,"mid":8,"high":8.6,"halfWidth":0.6,"basis":"editorial_prior","display":"7.4–8.6 (editorial prior)"},"scoreLayers":{"editorialPrior":7.95,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Public Agent Capability Benchmarks (SWE-bench, τ-bench, GAIA, OSWorld, WebArena, Terminal-Bench, BFCL, MLE-bench)."}],"disqualified":[],"expectedRegret":{"scoreDeltaVsUnconstrainedPick":0,"unconstrainedPickId":"promptfoo","activePrimaryId":"promptfoo","summary":"0.00 score regret: unconstrained editorial pick satisfies all caller constraints."},"evidenceAgeDays":{"editorialScoreAgeDays":1,"pricingVerifiedAgeDays":1,"editorialPriorAsOf":"2026-10-28","pricingLastVerifiedAt":"2026-10-28"},"confidenceBand":{"low":7.9,"mid":8.4,"high":8.9,"halfWidth":0.5,"basis":"editorial_prior","display":"7.9–8.9 (editorial prior)"},"constraintsApplied":false,"matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","constraintBundle":{"schemaVersion":"agentdepot.constraints.v1","task":null,"jtbd":"public-agent-benchmarks","constraintsRaw":"","constraintTokens":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"typedConstraints":[],"quotaExhausted":[],"degraded":[],"healthy":[],"budgetUsd":null,"maxLatencyMs":null,"region":null,"privacy":null,"exit":[],"existing":[],"existingSubscriptions":[],"requiredCapabilities":[],"switchingCost":null}},"freshness_state":"fresh","cite":"Depot Pick for public-agent-benchmarks: promptfoo (asOf 2026-10-28). See GET /api/jtbds/public-agent-benchmarks + GET /api/methodology.","asOf":"2026-10-28","honesty":["Recommended provider is the current Depot Pick from GET /api/decide. This quote does not change scores.","SKU rows come from data/pricing-rows.json via GET /api/pricing. Input and output stay on separate rows.","cheapestInput compares one native unit matching this job; non-token jobs never borrow chat $/1M rates.","pickIsPriceWinner is null when fewer than 2 candidates share a matching sourced unit price."],"links":{"decide":"/api/decide?jtbd=public-agent-benchmarks","quote":"/api/quote?jtbd=public-agent-benchmarks","pricing":"/api/pricing?provider=promptfoo","leaderboardPrice":"/api/leaderboards?jtbd=public-agent-benchmarks&sort=price","humanPage":"/jtbds/public-agent-benchmarks#quote","methodology":"/api/methodology"},"sourceConflicts":[],"volumeEstimateHint":"Add &volume=<number> for a plan-level monthly cost (GET /api/estimate?jtbd=public-agent-benchmarks&workload=...)"}