{"jtbd":{"id":"agent-regression-leaderboards","slug":"agent-regression-leaderboards","name":"Private Agent Regression Tracking, Version Scorecards & Internal Leaderboards","shortName":"Agent regression & scorecards"},"recommended":{"role":"depot_pick","id":"braintrust","slug":"braintrust","name":"Braintrust","website":"https://www.braintrust.dev","editorialOverall":8.29,"scoreKind":"curated_estimate","scoreAsOf":"2026-10-28","price":{"available":false,"unavailableReason":"no_sourced_price","skus":[],"normalizedUsdPerMillionTokens":null,"normalizationStatus":"unavailable","normalizationReason":"no_sourced_price"}},"priceComparison":{"skuKind":"none","unit":null,"comparison":"no comparable sourced unit for this job; chat input $/1M and other units were not borrowed","amountLabel":"Matching price","reportsUsdPerMillion":false,"matchedCandidates":0},"cheapestInput":null,"cheapest":null,"recommendedWorkloadQuote":{"providerId":"braintrust","providerName":"Braintrust","skuId":"braintrust-starter-plan","item":"Braintrust Starter Plan","plan":"Braintrust Starter Plan — Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - 10k scores + $2.50/1k - 14-day retention - Unlimited users, projects, datasets, playgrounds, and experiments","unit":"usd_per_month","amountUsd":0,"outputSkuId":null,"outputItem":null,"outputAmountUsd":null,"outputUnit":null,"sourceUrl":"https://www.braintrust.dev/pricing","asOf":"2026-10-21","pricingLastVerifiedAt":"2026-10-21","isFlagshipForJob":false,"differsFromFloorCheapest":false,"representativeQuote":{"skuId":"braintrust-starter-plan","item":"Braintrust Starter Plan","plan":"Braintrust Starter Plan — Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - 10k scores + $2.50/1k - 14-day retention - Unlimited users, projects, datasets, playgrounds, and experiments","amountUsd":0,"inputOrUnitUsd":0,"unit":"usd_per_month","outputUsd":null,"outputUnit":null,"sourceUrl":"https://www.braintrust.dev/pricing","asOf":"2026-10-21"},"estimatedTaskCost":{"tasks":1000,"retryAndToolOverheadMultiplier":1.15,"retryToolOverheadMultiplier":1.15,"unitsPerTask":1,"baseCostPer1kTasksUsd":0,"totalUsd":0,"estimatedTotalUsd":0,"perTaskUsd":0,"estimatedCostPerTaskUsd":0,"formula":"Monthly base plan 0/mo covering 1,000 tasks (1.15x overhead buffer)","assumptions":"Monthly subscription plan amortized over 1,000 tasks/month."}},"representativeQuote":{"skuId":"braintrust-starter-plan","item":"Braintrust Starter Plan","plan":"Braintrust Starter Plan — Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - 10k scores + $2.50/1k - 14-day retention - Unlimited users, projects, datasets, playgrounds, and experiments","amountUsd":0,"inputOrUnitUsd":0,"unit":"usd_per_month","outputUsd":null,"outputUnit":null,"sourceUrl":"https://www.braintrust.dev/pricing","asOf":"2026-10-21"},"estimatedTaskCost":{"tasks":1000,"retryAndToolOverheadMultiplier":1.15,"retryToolOverheadMultiplier":1.15,"unitsPerTask":1,"baseCostPer1kTasksUsd":0,"totalUsd":0,"estimatedTotalUsd":0,"perTaskUsd":0,"estimatedCostPerTaskUsd":0,"formula":"Monthly base plan 0/mo covering 1,000 tasks (1.15x overhead buffer)","assumptions":"Monthly subscription plan amortized over 1,000 tasks/month."},"cheapestGoodEnough":{"band":{"rule":"editorial overall >= Depot Pick overall - 0.5","minOverall":7.79,"pickOverall":8.29,"points":0.5},"pick":null,"reason":"no_matching_sourced_price","cheaperButBelowBand":[]},"dates":{"scoreAsOf":"2026-10-28","scoreKind":"curated_estimate","priceVerifiedAsOf":null,"note":"scoreAsOf is when the editorial score was last set; priceVerifiedAsOf is the newest fact-check of the pick's price rows. They are separate: a fresh price check does not re-score."},"sourcedInputs":[],"pickIsPriceWinner":null,"priceWinnerReason":"no_comparison_no_candidate_has_matching_sourced_price","priceWinnerNote":"Nothing was compared. no comparable sourced unit for this job; chat input $/1M and other units were not borrowed pickIsPriceWinner is null.","excluded":5,"coverage":{"candidates":6,"withAnySourcedRow":1,"withSourcedInputUsdPerMillion":0,"withMatchingPrice":0,"excludedNoSourcedPrice":5,"excludedFromCheapestInput":6},"normalizedUsdPerMillionTokens":null,"normalizationReason":"input_and_output_are_not_blended","seo":{"published":false,"reason":"Only 0 candidate(s) have a matching Matching price (none). Need 3 before a crawlable price section. Quote JSON is still returned."},"skipTopPick":false,"matchedConstraints":[],"unsupportedConstraints":[],"routePlan":{"jtbdId":"agent-regression-leaderboards","primary":{"providerId":"braintrust","name":"Braintrust","overall":8.29,"scoreBand":{"low":8,"mid":8.3,"high":8.7,"halfWidth":0.35,"basis":"verified_outcomes","display":"8.0–8.7 (verified outcomes)"},"scoreLayers":{"editorialPrior":8.29,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"primary","reason":"Editorial Depot Pick for Private Agent Regression Tracking, Version Scorecards & Internal Leaderboards; healthy/eligible under caller constraints."},"omittedReason":null,"fallback":[{"providerId":"langfuse","name":"Langfuse","overall":8.67,"scoreBand":{"low":8.1,"mid":8.7,"high":9.3,"halfWidth":0.6,"basis":"editorial_prior","display":"8.1–9.3 (editorial prior)"},"scoreLayers":{"editorialPrior":8.67,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Private Agent Regression Tracking, Version Scorecards & Internal Leaderboards."},{"providerId":"weights-biases","name":"Weights & Biases","overall":8.09,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.09,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Private Agent Regression Tracking, Version Scorecards & Internal Leaderboards."},{"providerId":"langsmith","name":"LangSmith","overall":8.16,"scoreBand":{"low":7.6,"mid":8.2,"high":8.8,"halfWidth":0.6,"basis":"editorial_prior","display":"7.6–8.8 (editorial prior)"},"scoreLayers":{"editorialPrior":8.16,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Private Agent Regression Tracking, Version Scorecards & Internal Leaderboards."},{"providerId":"comet-opik","name":"Comet Opik","overall":8.05,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.05,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Private Agent Regression Tracking, Version Scorecards & Internal Leaderboards."},{"providerId":"countersign","name":"Countersign","overall":8.74,"scoreBand":{"low":8.1,"mid":8.7,"high":9.3,"halfWidth":0.6,"basis":"editorial_prior","display":"8.1–9.3 (editorial prior)"},"scoreLayers":{"editorialPrior":8.74,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Private Agent Regression Tracking, Version Scorecards & Internal Leaderboards."}],"disqualified":[],"expectedRegret":{"scoreDeltaVsUnconstrainedPick":0,"unconstrainedPickId":"braintrust","activePrimaryId":"braintrust","summary":"0.00 score regret: unconstrained editorial pick satisfies all caller constraints."},"evidenceAgeDays":{"editorialScoreAgeDays":1,"pricingVerifiedAgeDays":1,"editorialPriorAsOf":"2026-10-28","pricingLastVerifiedAt":"2026-10-28"},"confidenceBand":{"low":8,"mid":8.3,"high":8.7,"halfWidth":0.35,"basis":"verified_outcomes","display":"8.0–8.7 (verified outcomes)"},"constraintsApplied":false,"matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","constraintBundle":{"schemaVersion":"agentdepot.constraints.v1","task":null,"jtbd":"agent-regression-leaderboards","constraintsRaw":"","constraintTokens":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"typedConstraints":[],"quotaExhausted":[],"degraded":[],"healthy":[],"budgetUsd":null,"maxLatencyMs":null,"region":null,"privacy":null,"exit":[],"existing":[],"existingSubscriptions":[],"requiredCapabilities":[],"switchingCost":null}},"freshness_state":"fresh","cite":"Depot Pick for agent-regression-leaderboards: braintrust (asOf 2026-10-28). See GET /api/jtbds/agent-regression-leaderboards + GET /api/methodology.","asOf":"2026-10-28","honesty":["Recommended provider is the current Depot Pick from GET /api/decide. This quote does not change scores.","SKU rows come from data/pricing-rows.json via GET /api/pricing. Input and output stay on separate rows.","cheapestInput compares one native unit matching this job; non-token jobs never borrow chat $/1M rates.","pickIsPriceWinner is null when fewer than 2 candidates share a matching sourced unit price."],"links":{"decide":"/api/decide?jtbd=agent-regression-leaderboards","quote":"/api/quote?jtbd=agent-regression-leaderboards","pricing":"/api/pricing?provider=braintrust","leaderboardPrice":"/api/leaderboards?jtbd=agent-regression-leaderboards&sort=price","humanPage":"/jtbds/agent-regression-leaderboards#quote","methodology":"/api/methodology"},"sourceConflicts":[{"providerId":"langsmith","field":"calculator monthly total: static '$49 / mo' in the page HTML vs the calculator script's own default (5,000 traces, 1 seat → $39)","sourceUrl":"https://www.langchain.com/pricing","asOf":"2026-10-22","quotes":["'Your agent stack Monthly total LSU 0 $49 / mo Estimate only. Actual LSU consumption varies with the work your agents perform.' (https://www.langchain.com/pricing, server-rendered HTML; WebFetch renders the same figure as \"$49/ mo\")","'traceLsuPerTrace:0.005, seatPrice:39, tracesFreeIncluded:10000,' … defaults 'traces:5000,' 'seats:1,' … 'return {lsu:lsu,cost:(seats||1)*RATES.seatPrice+lsu*RATES.lsuPrice};' (page calculator script: 1 × $39 + max(0, 5,000 − 10,000) × 0.005 LSU = $39); plan table 'Add unlimited seats $39 per seat/month'"],"safeAssumption":"Use the published rates ($39 per Plus seat, 0.005 LSU per base trace, 1 LSU = $1), which the plan cards, the comparison table, the tooltip and the calculator script all agree on; the $49 static figure is not reproducible from any published rate and is not used."},{"providerId":"comet-opik","field":"Opik Cloud team-member limits: plan cards 'Up to 10 team members' (Free Cloud) / 'Up to 50 team members' (Pro Cloud) vs FAQ 'All plans include unlimited team members'","sourceUrl":"https://www.comet.com/site/pricing/","asOf":"2026-10-22","quotes":["'Free Cloud Perfect for individuals $0 Free plan Get Started Up to 10 team members 25k spans per month 60-day data retention' … 'Pro Cloud Popular Expanded usage for teams $19 Per month Start Free Trial Up to 50 team members 100k spans per month 60-day data retention' … 'Enterprise Security, compliance & flexible deployments Custom Contact Us Unlimited team members Custom usage plans' (https://www.comet.com/site/pricing/ plan cards)","'What are the differences between Opik Open Source, Opik Free, Opik Pro, and Opik Enterprise? The main differences are in usage limits and data retention. All plans include unlimited team members and the full suite of LLM observability and evaluation features.' (https://www.comet.com/site/pricing/ Questions & Answers)"],"safeAssumption":"No price impact: no per-member price is published on any plan. Depot uses the stricter card caps as plan ceilings (opik_team_members maxTotal 10 on Free Cloud, 50 on Pro Cloud), so more than 50 members leaves Pro uncosted rather than assuming unlimited; the FAQ wording is not used."}],"volumeEstimateHint":"Add &volume=<number> for a plan-level monthly cost (GET /api/estimate?jtbd=agent-regression-leaderboards&workload=...)"}