{"evidenceStatus":{"decisionRole":"independent_prior_and_candidate_generator","positioning":"The independent, agent-readable prior for choosing AI infrastructure — not a universal router, generic marketplace, or synthetic leaderboard.","freshnessThresholdDays":30,"freshness_policy":{"policy_version":"agentdepot.freshness_policy.v1","threshold_days":30},"freshness_state":"fresh","editorial_freshness_state":"fresh","unqualified_current_pick_allowed":true,"pick_qualification":"qualified_current_pick","promotionClass":"verified_outcome_backed","evidenceClass":"editorial_plus_verified_receipt","staleDowngraded":false,"effectiveConfidence":"high_verified","confidenceHalfWidth":0.35,"pickScoreAgeDays":0,"editorialJudgmentAgeDays":9,"pricingAgeDays":null,"isPickScoreStale":false,"isPricingStale":true,"editorialJudgmentAsOf":"2026-10-19","pricingVerifiedAt":null,"outcomeVerifiedAt":"2026-10-28","effectiveVerifiedAsOf":"2026-10-28","downgradeReasons":[],"stalenessWarning":"Editorial judgment and deterministic pricing verification are both within the 30-day freshness threshold.","promotionGate":{"deterministicPricingCheck":false,"approvedScoreReaffirm":true,"pendingHumanReviewBlock":false,"zeroUnverifiedSkus":true,"costEfficiencyConsistent":true,"lockInAndReliabilityConsistent":true,"secondModelOrRuleAudit":true,"countersignReceiptIds":["cs-54379c97","cs-6eab5b5d","cs-316f2587","cs-0e63df18","cs-1035466f"],"otsAttestationHead":"b32634d5784cbfe663be0a16cce4f60a3ff1c3d4490437be00361cb4ae5d5d59","sourceCheckDate":"2026-10-27","activePricingRowsCount":0,"freeResourcesCount":0,"runnerUpPricingRowsCount":1},"claimProvenance":{"claim_id":"claim:llm-as-judge-calibration:braintrust:depot-pick","provider_id":"braintrust","jtbd_id":"llm-as-judge-calibration","primary_url":"https://www.braintrust.dev/docs","source_type":"primary_docs","source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.braintrust.dev","final_url":"https://www.braintrust.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:fdb47a7e8e944079","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/docs","final_url":"https://www.braintrust.dev/docs","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_docs","is_claim_specific":true,"source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/pricing","final_url":"https://www.braintrust.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:60bb11c5a2c023e3","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"verifiedOutcomeSummary":{"totalVerifiedReceipts":0,"byEvidenceTier":{"E0_ASSERTION":0,"E1_SOURCE_BOUND":0,"E2_DELIVERY":0,"E3_INDEPENDENT_OUTCOME":0,"E4_REPLICATED_OUTCOME":0},"bySubjectLayer":{"provider":0,"product":0,"model_version":0,"runtime_route":0}}},"freshness_state":"fresh","freshness_policy":{"policy_version":"agentdepot.freshness_policy.v1","threshold_days":30},"runtimeRoutes":[{"routeId":"llm-as-judge-calibration:braintrust:primary","roleForJob":"depot_pick","provider":{"id":"braintrust","slug":"braintrust","name":"Braintrust","website":"https://www.braintrust.dev","status":"active"},"model":{"representativeModelId":"Braintrust Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.braintrust.dev","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.braintrust.dev (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Braintrust Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"Starter: Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - ","subscriptionFirstEligible":true,"documentedPlansCount":2},"regions":{"supportedRegions":["us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.6,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"patronus"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:managed_cloud_service:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}},{"routeId":"llm-as-judge-calibration:patronus:primary","roleForJob":"runner_up","provider":{"id":"patronus","slug":"patronus","name":"Patronus AI","website":"https://www.patronus.ai","status":"active"},"model":{"representativeModelId":"Patronus AI Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.patronus.ai","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.patronus.ai (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Patronus AI Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=patronus","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global","us"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":false,"parallelToolCalls":false,"mcpServerAvailable":false,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.5,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"braintrust"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:patronus:patronus-primary:managed_cloud_service:global","provider_id":"patronus","product_id":"patronus-core","model_id":"patronus-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}}],"verifiedOutcomes":[],"jtbd":{"id":"llm-as-judge-calibration","slug":"llm-as-judge-calibration","name":"LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation","shortName":"LLM-as-judge calibration","problemStatement":"Calibrate LLM-as-judge graders against human gold labels and deterministic assertions so your eval harness does not reward verbose, sycophantic, or self-preferring outputs."},"pick":{"id":"braintrust","slug":"braintrust","name":"Braintrust","tagline":"Depot pick when evals are the product quality gate. Pair with Helicone/Langfuse for cheap request logs if…","website":"https://www.braintrust.dev","overall":8.32,"scoreBand":{"low":8,"mid":8.3,"high":8.7,"halfWidth":0.35,"basis":"verified_outcomes","display":"8.0–8.7 (verified outcomes)"},"scoreLayers":{"editorialPrior":8.32,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","pricingVerifiedAsOf":"2026-10-28","scoreVerifiedAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","liveHealth":{"status":"operational","checkedAt":"2026-10-28T15:04:27.127Z","statusPageUrl":"https://status.braintrust.dev","latencyMs":1315,"indicator":"none","description":"Public status/endpoint reachable (HTTP 200)"},"scoreRange":[7.97,8.67],"formattedScore":"8.3 ± 0.35 (8.0–8.7 · editorial_plus_verified_receipt)","evidenceClass":"editorial_plus_verified_receipt","promotionClass":"verified_outcome_backed","staleDowngraded":false,"effectiveConfidence":"high_verified","confidenceInterval":{"low":7.97,"mid":8.32,"high":8.67,"halfWidth":0.35,"confidenceLevel":0.8,"basis":"narrowed_verified_outcome_interval_0.35_backed_by_countersign_receipts"},"sourceAgeDays":9,"pricingAgeDays":null,"freshness_state":"fresh","dissentingEvidence":["Broader tracing competitors exist","Price entry higher than hobby tools","Learning curve for eval design","When not to use top pick: Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","Runner-up Patronus AI (patronus) is the primary alternative when constraints disqualify Braintrust."],"whatWouldChangePick":["Caller constraint matching: Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","Live quota or rate-limit exhaustion on braintrust (pass ?exhausted=braintrust or liveQuota.exhaustedProviders=[\"braintrust\"] to promote patronus).","Active subscription or enterprise commit on patronus combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.braintrust.dev or unverified SKU drift detected by npm run check:sources."],"claimProvenance":{"claim_id":"claim:llm-as-judge-calibration:braintrust:depot-pick","provider_id":"braintrust","jtbd_id":"llm-as-judge-calibration","primary_url":"https://www.braintrust.dev/docs","source_type":"primary_docs","source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.braintrust.dev","final_url":"https://www.braintrust.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:fdb47a7e8e944079","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/docs","final_url":"https://www.braintrust.dev/docs","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_docs","is_claim_specific":true,"source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/pricing","final_url":"https://www.braintrust.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:60bb11c5a2c023e3","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"calibratedScore":{"point":8.32,"band":[7.97,8.67],"interval":{"lower":7.97,"upper":8.67,"half_width":0.35},"score_kind":"experimental_calibrated_band","evidence_class":"experimental","confidence":"low","confidence_ceiling":"high","age_days":0,"dissent":["Broader tracing competitors exist","Price entry higher than hobby tools","Learning curve for eval design","When not to use top pick: Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","Runner-up Patronus AI (patronus) is the primary alternative when constraints disqualify Braintrust."],"what_would_change_this":["Caller constraint matching: Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","Live quota or rate-limit exhaustion on braintrust (pass ?exhausted=braintrust or liveQuota.exhaustedProviders=[\"braintrust\"] to promote patronus).","Active subscription or enterprise commit on patronus combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.braintrust.dev or unverified SKU drift detected by npm run check:sources."],"subject_layer":"provider"},"subject_ref":{"layer":"provider","provider_id":"braintrust","product_id":"braintrust-core","model_id":null,"version":null,"route_class":null,"inheritance_policy":"no_cross_layer_inheritance"},"pricingNotes":"Eval-first platform; data-based pricing with unlimited seats on many plans; strong CI eval story.","scores":{"capabilityFit":9.2,"pricePerformance":7.5,"latencyReliability":8.6,"developerDx":8.8,"privacyPosture":8,"lockInRisk":5,"maturity":8.2,"overall":8.32,"kind":"curated_estimate","confidence":"high","asOf":"2026-10-28"}},"liveHealth":{"status":"operational","checkedAt":"2026-10-28T15:04:27.127Z","statusPageUrl":"https://status.braintrust.dev","latencyMs":1315,"indicator":"none","description":"Public status/endpoint reachable (HTTP 200)"},"selectedPick":{"id":"braintrust","slug":"braintrust","name":"Braintrust","tagline":"Depot pick when evals are the product quality gate. Pair with Helicone/Langfuse for cheap request logs if…","website":"https://www.braintrust.dev","overall":8.32,"scoreBand":{"low":8,"mid":8.3,"high":8.7,"halfWidth":0.35,"basis":"verified_outcomes","display":"8.0–8.7 (verified outcomes)"},"scoreLayers":{"editorialPrior":8.32,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","pricingVerifiedAsOf":"2026-10-28","scoreVerifiedAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","liveHealth":{"status":"operational","checkedAt":"2026-10-28T15:04:27.127Z","statusPageUrl":"https://status.braintrust.dev","latencyMs":1315,"indicator":"none","description":"Public status/endpoint reachable (HTTP 200)"},"scoreRange":[7.97,8.67],"formattedScore":"8.3 ± 0.35 (8.0–8.7 · editorial_plus_verified_receipt)","evidenceClass":"editorial_plus_verified_receipt","promotionClass":"verified_outcome_backed","staleDowngraded":false,"effectiveConfidence":"high_verified","confidenceInterval":{"low":7.97,"mid":8.32,"high":8.67,"halfWidth":0.35,"confidenceLevel":0.8,"basis":"narrowed_verified_outcome_interval_0.35_backed_by_countersign_receipts"},"sourceAgeDays":9,"pricingAgeDays":null,"freshness_state":"fresh","pricingNotes":"Eval-first platform; data-based pricing with unlimited seats on many plans; strong CI eval story.","scores":{"capabilityFit":9.2,"pricePerformance":7.5,"latencyReliability":8.6,"developerDx":8.8,"privacyPosture":8,"lockInRisk":5,"maturity":8.2,"overall":8.32,"kind":"curated_estimate","confidence":"high","asOf":"2026-10-28"}},"editorialPick":{"id":"braintrust","slug":"braintrust","name":"Braintrust","tagline":"Depot pick when evals are the product quality gate. Pair with Helicone/Langfuse for cheap request logs if…","website":"https://www.braintrust.dev","overall":8.32,"scoreBand":{"low":8,"mid":8.3,"high":8.7,"halfWidth":0.35,"basis":"verified_outcomes","display":"8.0–8.7 (verified outcomes)"},"scoreLayers":{"editorialPrior":8.32,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","pricingVerifiedAsOf":"2026-10-28","scoreVerifiedAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","liveHealth":{"status":"operational","checkedAt":"2026-10-28T15:04:27.127Z","statusPageUrl":"https://status.braintrust.dev","latencyMs":1315,"indicator":"none","description":"Public status/endpoint reachable (HTTP 200)"},"scoreRange":[7.97,8.67],"formattedScore":"8.3 ± 0.35 (8.0–8.7 · editorial_plus_verified_receipt)","evidenceClass":"editorial_plus_verified_receipt","promotionClass":"verified_outcome_backed","staleDowngraded":false,"effectiveConfidence":"high_verified","confidenceInterval":{"low":7.97,"mid":8.32,"high":8.67,"halfWidth":0.35,"confidenceLevel":0.8,"basis":"narrowed_verified_outcome_interval_0.35_backed_by_countersign_receipts"},"sourceAgeDays":9,"pricingAgeDays":null,"freshness_state":"fresh","pricingNotes":"Eval-first platform; data-based pricing with unlimited seats on many plans; strong CI eval story.","scores":{"capabilityFit":9.2,"pricePerformance":7.5,"latencyReliability":8.6,"developerDx":8.8,"privacyPosture":8,"lockInRisk":5,"maturity":8.2,"overall":8.32,"kind":"curated_estimate","confidence":"high","asOf":"2026-10-28"}},"disqualifiedCandidates":[],"runnersUp":[{"id":"patronus","slug":"patronus","name":"Patronus AI","tagline":"Layer on when regulated hallucination/safety gates matter more than general CI evals. Lakera for injection…","website":"https://www.patronus.ai","overall":7.43,"scoreBand":{"low":6.8,"mid":7.4,"high":8,"halfWidth":0.6,"basis":"editorial_prior","display":"6.8–8.0 (editorial prior)"},"scoreLayers":{"editorialPrior":7.43,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","pricingVerifiedAsOf":"2026-10-28","scoreVerifiedAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","liveHealth":{"status":"unknown","checkedAt":"2026-10-28T15:04:28.744Z","statusPageUrl":"https://www.patronus.ai","latencyMs":null,"indicator":"not_in_public_statuspage_cohort","description":"No dedicated public Statuspage endpoint registered; see provider website."},"scoreRange":[6.93,7.93],"formattedScore":"7.4 ± 0.5 (6.9–7.9 · editorial_prior)","evidenceClass":"editorial_prior","promotionClass":"deterministic_source_reverified","staleDowngraded":false,"effectiveConfidence":"medium","confidenceInterval":{"low":6.93,"mid":7.43,"high":7.93,"halfWidth":0.5,"confidenceLevel":0.8,"basis":"deterministic_source_reverified_interval_0.5"},"sourceAgeDays":9,"pricingAgeDays":null,"freshness_state":"fresh","pricingNotes":"As of 2026-10-28: Enterprise evaluation/safety platform (Lynx hallucination detector, Glider judge). Public list price thin — secondary reports cite ~$25/mo Base + per-1k evaluator calls; treat as sales-led until official card pinned. Series-B backed.","scores":{"capabilityFit":8.4,"pricePerformance":5.5,"latencyReliability":8,"developerDx":7.2,"privacyPosture":8.5,"lockInRisk":5.5,"maturity":7.5,"overall":7.43,"kind":"curated_estimate","confidence":"medium","asOf":"2026-10-28"}},{"id":"ragas","slug":"ragas","name":"Ragas","tagline":"Default RAG metric library. Pair with Promptfoo/Braintrust for broader agent evals.","website":"https://docs.ragas.io","overall":8.3,"scoreBand":{"low":7.7,"mid":8.3,"high":8.9,"halfWidth":0.6,"basis":"editorial_prior","display":"7.7–8.9 (editorial prior)"},"scoreLayers":{"editorialPrior":8.3,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","pricingVerifiedAsOf":"2026-10-28","scoreVerifiedAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","liveHealth":{"status":"unknown","checkedAt":"2026-10-28T15:04:28.744Z","statusPageUrl":"https://docs.ragas.io","latencyMs":null,"indicator":"not_in_public_statuspage_cohort","description":"No dedicated public Statuspage endpoint registered; see provider website."},"scoreRange":[7.8,8.8],"formattedScore":"8.3 ± 0.5 (7.8–8.8 · editorial_prior)","evidenceClass":"editorial_prior","promotionClass":"deterministic_source_reverified","staleDowngraded":false,"effectiveConfidence":"high","confidenceInterval":{"low":7.8,"mid":8.3,"high":8.8,"halfWidth":0.5,"confidenceLevel":0.8,"basis":"deterministic_source_reverified_interval_0.5"},"sourceAgeDays":9,"pricingAgeDays":null,"freshness_state":"fresh","pricingNotes":"As of 2026-10-28: Free open-source RAG evaluation library; cost is judge-LLM tokens only (~$0.50–$2 per 100 queries with cheap judges). No hosted SaaS required. Deep LangChain/LlamaIndex integration.","scores":{"capabilityFit":8,"pricePerformance":9.5,"latencyReliability":7.5,"developerDx":8.2,"privacyPosture":9.2,"lockInRisk":2,"maturity":7.8,"overall":8.3,"kind":"curated_estimate","confidence":"high","asOf":"2026-10-28"}},{"id":"galileo","slug":"galileo","name":"Galileo","tagline":"Pick for production hallucination measurement + guardrails. Braintrust/Promptfoo for lighter eval loops.","website":"https://www.galileo.ai","overall":7.57,"scoreBand":{"low":7,"mid":7.6,"high":8.2,"halfWidth":0.6,"basis":"editorial_prior","display":"7.0–8.2 (editorial prior)"},"scoreLayers":{"editorialPrior":7.57,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","pricingVerifiedAsOf":"2026-10-28","scoreVerifiedAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","liveHealth":{"status":"unknown","checkedAt":"2026-10-28T15:04:28.744Z","statusPageUrl":"https://www.galileo.ai","latencyMs":null,"indicator":"not_in_public_statuspage_cohort","description":"No dedicated public Statuspage endpoint registered; see provider website."},"scoreRange":[7.07,8.07],"formattedScore":"7.6 ± 0.5 (7.1–8.1 · editorial_prior)","evidenceClass":"editorial_prior","promotionClass":"deterministic_source_reverified","staleDowngraded":false,"effectiveConfidence":"high","confidenceInterval":{"low":7.07,"mid":7.57,"high":8.07,"halfWidth":0.5,"confidenceLevel":0.8,"basis":"deterministic_source_reverified_interval_0.5"},"sourceAgeDays":9,"pricingAgeDays":null,"freshness_state":"fresh","pricingNotes":"As of 2026-10-28 (galileo.ai/pricing live): Free **$0/mo** — 5,000 traces/mo, unlimited users/custom evals. Pro **$100/mo** (annual; scales with traces) — 50,000 traces/mo, standard RBAC, advanced analytics, Slack support. Enterprise contact — unlimited traces, VPC/on-prem, SSO, real-time guardrails, 24/7.","scores":{"capabilityFit":8.4,"pricePerformance":6.2,"latencyReliability":8,"developerDx":7.8,"privacyPosture":8,"lockInRisk":6,"maturity":7.8,"overall":7.57,"kind":"curated_estimate","confidence":"high","asOf":"2026-10-28"}},{"id":"confident-ai","slug":"confident-ai","name":"Confident AI","tagline":"Best when your team already thinks in pytest/DeepEval. Braintrust remains Depot hosted-evals default…","website":"https://www.confident-ai.com","overall":8.05,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.05,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","pricingVerifiedAsOf":"2026-10-28","scoreVerifiedAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","liveHealth":{"status":"unknown","checkedAt":"2026-10-28T15:04:28.744Z","statusPageUrl":"https://www.confident-ai.com","latencyMs":null,"indicator":"not_in_public_statuspage_cohort","description":"No dedicated public Statuspage endpoint registered; see provider website."},"scoreRange":[7.55,8.55],"formattedScore":"8.1 ± 0.5 (7.5–8.6 · editorial_prior)","evidenceClass":"editorial_prior","promotionClass":"deterministic_source_reverified","staleDowngraded":false,"effectiveConfidence":"high","confidenceInterval":{"low":7.55,"mid":8.05,"high":8.55,"halfWidth":0.5,"confidenceLevel":0.8,"basis":"deterministic_source_reverified_interval_0.5"},"sourceAgeDays":9,"pricingAgeDays":null,"freshness_state":"fresh","pricingNotes":"As of 2026-10-28 (confident-ai.com/pricing): Free forever (2 seats, 1 project, 5 test runs/week, 1 GB-mo traces). Starter $200/mo (5 projects, 5 GB-mo then $1/GB-mo). Team $2,000/mo (unlimited projects, 75 GB-mo). Enterprise custom (on-prem, HIPAA, residency). Online eval tokens ~$0.05/$0.40 per 1M in/out.","scores":{"capabilityFit":8.6,"pricePerformance":7.8,"latencyReliability":8,"developerDx":9,"privacyPosture":7.8,"lockInRisk":4.5,"maturity":7.2,"overall":8.05,"kind":"curated_estimate","confidence":"high","asOf":"2026-10-28"}},{"id":"promptfoo","slug":"promptfoo","name":"Promptfoo","tagline":"Default OSS eval harness. Braintrust when you want hosted collaboration; Promptfoo when you want local/CI…","website":"https://www.promptfoo.dev","overall":8.41,"scoreBand":{"low":7.8,"mid":8.4,"high":9,"halfWidth":0.6,"basis":"editorial_prior","display":"7.8–9.0 (editorial prior)"},"scoreLayers":{"editorialPrior":8.41,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","pricingVerifiedAsOf":"2026-10-28","scoreVerifiedAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","liveHealth":{"status":"unknown","checkedAt":"2026-10-28T15:04:28.744Z","statusPageUrl":"https://www.promptfoo.dev","latencyMs":null,"indicator":"not_in_public_statuspage_cohort","description":"No dedicated public Statuspage endpoint registered; see provider website."},"scoreRange":[7.91,8.91],"formattedScore":"8.4 ± 0.5 (7.9–8.9 · editorial_prior)","evidenceClass":"editorial_prior","promotionClass":"deterministic_source_reverified","staleDowngraded":false,"effectiveConfidence":"high","confidenceInterval":{"low":7.91,"mid":8.41,"high":8.91,"halfWidth":0.5,"confidenceLevel":0.8,"basis":"deterministic_source_reverified_interval_0.5"},"sourceAgeDays":9,"pricingAgeDays":null,"freshness_state":"fresh","pricingNotes":"As of 2026-10-24 (promptfoo.dev/pricing, fetched 2026-10-24): Community 'Free Forever' ('Our open-source tool, perfect for individual developers and small teams.') with 'Red teaming (10k probes/month)' and 'Run locally or self-host on your own infrastructure'; 'A probe is a single request made to your target system during red team testing.' Enterprise 'Custom' ('Contact us for a personalized quote.'; 'Enterprise customers can purchase additional probes for larger scale testing.'; 'Managed cloud deployment'). On-Premise 'Custom' ('Deployment on your own infrastructure'). No list price is published, so Enterprise price stays null.","scores":{"capabilityFit":8.2,"pricePerformance":9.2,"latencyReliability":7.8,"developerDx":8.8,"privacyPosture":9,"lockInRisk":2.5,"maturity":8,"overall":8.41,"kind":"curated_estimate","confidence":"high","asOf":"2026-10-28"}}],"scoreBand":{"low":8,"mid":8.3,"high":8.7,"halfWidth":0.35,"basis":"verified_outcomes","display":"8.0–8.7 (verified outcomes)"},"scoreLayers":{"editorialPrior":8.32,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"whyPickOverHigherScore":{"pickId":"braintrust","pickName":"Braintrust","pickScore":8.32,"highestScoredCandidateId":"promptfoo","highestScoredCandidateName":"Promptfoo","highestScore":8.41,"pickHasLowerCompositeScore":true,"delta":-0.09,"pickLeadsOnDimensions":["capabilityFit (9.2 vs 8.2)","latencyReliability (8.6 vs 7.8)","maturity (8.2 vs 8)"],"explanation":"Braintrust (8.32) is the editorial pick for LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation over higher-composite Promptfoo (8.41) because composite scores are provider-wide priors while Braintrust wins on capabilityFit (9.2 vs 8.2), latencyReliability (8.6 vs 7.8), maturity (8.2 vs 8): Braintrust lets teams version code-based and LLM-judge scorers side-by-side, measure judge agreement against human review queues on the same dataset rows, and swap judge models (e.g. grading an OpenAI agent with Anthropic or vice versa) in one config line."},"cheapest":null,"recommendedWorkloadQuote":{"providerId":"braintrust","providerName":"Braintrust","skuId":"braintrust-starter-plan","item":"Braintrust Starter Plan","plan":"Braintrust Starter Plan — Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - 10k scores + $2.50/1k - 14-day retention - Unlimited users, projects, datasets, playgrounds, and experiments","unit":"usd_per_month","amountUsd":0,"outputSkuId":null,"outputItem":null,"outputAmountUsd":null,"outputUnit":null,"sourceUrl":"https://www.braintrust.dev/pricing","asOf":"2026-10-21","pricingLastVerifiedAt":"2026-10-21","isFlagshipForJob":false,"differsFromFloorCheapest":false,"representativeQuote":{"skuId":"braintrust-starter-plan","item":"Braintrust Starter Plan","plan":"Braintrust Starter Plan — Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - 10k scores + $2.50/1k - 14-day retention - Unlimited users, projects, datasets, playgrounds, and experiments","amountUsd":0,"inputOrUnitUsd":0,"unit":"usd_per_month","outputUsd":null,"outputUnit":null,"sourceUrl":"https://www.braintrust.dev/pricing","asOf":"2026-10-21"},"estimatedTaskCost":{"tasks":1000,"retryAndToolOverheadMultiplier":1.15,"retryToolOverheadMultiplier":1.15,"unitsPerTask":1,"baseCostPer1kTasksUsd":0,"totalUsd":0,"estimatedTotalUsd":0,"perTaskUsd":0,"estimatedCostPerTaskUsd":0,"formula":"Monthly base plan 0/mo covering 1,000 tasks (1.15x overhead buffer)","assumptions":"Monthly subscription plan amortized over 1,000 tasks/month."}},"representativeQuote":{"skuId":"braintrust-starter-plan","item":"Braintrust Starter Plan","plan":"Braintrust Starter Plan — Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - 10k scores + $2.50/1k - 14-day retention - Unlimited users, projects, datasets, playgrounds, and experiments","amountUsd":0,"inputOrUnitUsd":0,"unit":"usd_per_month","outputUsd":null,"outputUnit":null,"sourceUrl":"https://www.braintrust.dev/pricing","asOf":"2026-10-21"},"estimatedTaskCost":{"tasks":1000,"retryAndToolOverheadMultiplier":1.15,"retryToolOverheadMultiplier":1.15,"unitsPerTask":1,"baseCostPer1kTasksUsd":0,"totalUsd":0,"estimatedTotalUsd":0,"perTaskUsd":0,"estimatedCostPerTaskUsd":0,"formula":"Monthly base plan 0/mo covering 1,000 tasks (1.15x overhead buffer)","assumptions":"Monthly subscription plan amortized over 1,000 tasks/month."},"freeAlternatives":[],"referenceDocs":[],"falsification":{"jtbdId":"llm-as-judge-calibration","jtbdName":"LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation","currentPickId":"braintrust","currentPickScore":8.32,"challengerId":"patronus","challengerScore":7.43,"currentScoreGap":0.89,"minVerifiedOutcomes":10,"requiredOutcomeMargin":1.04,"requiredSuccessRateDelta":0.15,"maxPickPriceIncreasePct":25,"whatWouldChangePick":["Patronus AI (patronus) outperforms Braintrust (braintrust) by >= 1.04 outcome points (or >= 15% worked-rate delta) across >= 10 Countersign-verified production outcomes.","A verified price increase > 25% on Braintrust's representative SKU for llm-as-judge-calibration without a corresponding capability gain.","Live caller rail exhaustion (?exhausted=braintrust) or sustained >= 30% degraded/failed rate in verified production outcomes."],"frozenHoldoutPolicy":{"holdoutSplit":"20% frozen benchmark holdout tasks + 80% live production outcome receipts","requireCountersignReceipt":true,"requiredEvidenceTiers":["outcome_verified","reference_checked"],"minDistinctContributors":2,"antiGamingRule":"Unverified self-reports and seed/editorial rows never mutate picks or scores."},"triggers":[{"kind":"verified_outcomes_outperformance","challengerId":"patronus","minVerifiedOutcomes":10,"requiredOutcomeMargin":1.04,"requiredWorkedRateDelta":0.15},{"kind":"verified_price_increase","providerId":"braintrust","maxPriceIncreasePct":25},{"kind":"live_rail_exhaustion_or_degradation","providerId":"braintrust","immediateOverrideUrl":"/api/decide?jtbd=llm-as-judge-calibration&exhausted=braintrust"}],"submissionSpec":{"falsifyEndpoint":"/api/falsify?jtbd=llm-as-judge-calibration","outcomesEndpoint":"POST /api/feedback","correctionsEndpoint":"POST /api/corrections","payloadTemplate":{"agentIdentity":"agent:<your-agent-id>","jtbdId":"llm-as-judge-calibration","providerId":"patronus","outcome":"worked","detail":"Comparative production evaluation on llm-as-judge-calibration (n>=10 tasks) with Countersign receipt.","confidence":"high","receipt":{"countersign":{"claim_id":"cs-00000000","status":"verified","evidence_tier":"outcome_verified","counts_as_verified":true}}}}},"whatWouldChangePick":["Patronus AI (patronus) outperforms Braintrust (braintrust) by >= 1.04 outcome points (or >= 15% worked-rate delta) across >= 10 Countersign-verified production outcomes.","A verified price increase > 25% on Braintrust's representative SKU for llm-as-judge-calibration without a corresponding capability gain.","Live caller rail exhaustion (?exhausted=braintrust) or sustained >= 30% degraded/failed rate in verified production outcomes."],"routePlan":{"jtbdId":"llm-as-judge-calibration","primary":{"providerId":"braintrust","name":"Braintrust","overall":8.32,"scoreBand":{"low":8,"mid":8.3,"high":8.7,"halfWidth":0.35,"basis":"verified_outcomes","display":"8.0–8.7 (verified outcomes)"},"scoreLayers":{"editorialPrior":8.32,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"primary","reason":"Editorial Depot Pick for LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation; healthy/eligible under caller constraints."},"omittedReason":null,"fallback":[{"providerId":"patronus","name":"Patronus AI","overall":7.43,"scoreBand":{"low":6.8,"mid":7.4,"high":8,"halfWidth":0.6,"basis":"editorial_prior","display":"6.8–8.0 (editorial prior)"},"scoreLayers":{"editorialPrior":7.43,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation."},{"providerId":"ragas","name":"Ragas","overall":8.3,"scoreBand":{"low":7.7,"mid":8.3,"high":8.9,"halfWidth":0.6,"basis":"editorial_prior","display":"7.7–8.9 (editorial prior)"},"scoreLayers":{"editorialPrior":8.3,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation."},{"providerId":"galileo","name":"Galileo","overall":7.57,"scoreBand":{"low":7,"mid":7.6,"high":8.2,"halfWidth":0.6,"basis":"editorial_prior","display":"7.0–8.2 (editorial prior)"},"scoreLayers":{"editorialPrior":7.57,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation."},{"providerId":"confident-ai","name":"Confident AI","overall":8.05,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.05,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation."},{"providerId":"promptfoo","name":"Promptfoo","overall":8.41,"scoreBand":{"low":7.8,"mid":8.4,"high":9,"halfWidth":0.6,"basis":"editorial_prior","display":"7.8–9.0 (editorial prior)"},"scoreLayers":{"editorialPrior":8.41,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation."}],"disqualified":[],"expectedRegret":{"scoreDeltaVsUnconstrainedPick":0,"unconstrainedPickId":"braintrust","activePrimaryId":"braintrust","summary":"0.00 score regret: unconstrained editorial pick satisfies all caller constraints."},"evidenceAgeDays":{"editorialScoreAgeDays":1,"pricingVerifiedAgeDays":1,"editorialPriorAsOf":"2026-10-28","pricingLastVerifiedAt":"2026-10-28"},"confidenceBand":{"low":8,"mid":8.3,"high":8.7,"halfWidth":0.35,"basis":"verified_outcomes","display":"8.0–8.7 (verified outcomes)"},"constraintsApplied":false,"matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","constraintBundle":{"schemaVersion":"agentdepot.constraints.v1","task":null,"jtbd":"llm-as-judge-calibration","constraintsRaw":"","constraintTokens":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"typedConstraints":[],"quotaExhausted":[],"degraded":[],"healthy":[],"budgetUsd":null,"maxLatencyMs":null,"region":null,"privacy":null,"exit":[],"existing":[],"existingSubscriptions":[],"requiredCapabilities":[],"switchingCost":null}},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","tie":false,"scoreGap":0.89,"scoreMargin":0.89,"isNearTie":false,"pickConfidenceBand":"clear_leader","nearTie":{"isNearTie":false,"scoreMargin":0.89,"threshold":0.35,"pickConfidenceBand":"clear_leader","coPickProviderId":"patronus","coPickProviderName":"Patronus AI","coPickScore":7.43,"decidingFactor":null,"methodologyWarning":null,"tieBreakerRule":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo."},"decidingFactor":null,"tieNote":null,"rationale":"Braintrust lets teams version code-based and LLM-judge scorers side-by-side, measure judge agreement against human review queues on the same dataset rows, and swap judge models (e.g. grading an OpenAI agent with Anthropic or vice versa) in one config line.","whenNotTopPick":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","whenNotTopPickRules":[{"ifConstraint":"escape_hatch_clause_1","recommendedProviderId":"patronus","activeModelOrSku":"patronus-standard","reason":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI"},{"ifConstraint":"escape_hatch_clause_2","recommendedProviderId":"ragas","activeModelOrSku":"ragas-oss","reason":"research-grade RAG faithfulness/context metrics — Ragas"},{"ifConstraint":"escape_hatch_clause_3","recommendedProviderId":"galileo","activeModelOrSku":"galileo-evaluate","reason":"enterprise guardrail + hallucination scoring — Galileo"},{"ifConstraint":"escape_hatch_clause_4","recommendedProviderId":"confident-ai","activeModelOrSku":"confident-ai-standard","reason":"pytest-native G-Eval/DAG rubrics — Confident AI"},{"ifConstraint":"exit:self_host_or_open_weights","recommendedProviderId":"promptfoo","activeModelOrSku":"promptfoo-oss","reason":"OSS local assertion matrices — Promptfoo."}],"bootstrap":{"authType":"api_key","envVars":["BRAINTRUST_API_KEY"],"signupUrl":"https://www.braintrust.dev/signup","apiKeyUrl":"https://www.braintrust.dev/app/settings?subroute=api-keys","docsUrl":"https://www.braintrust.dev/docs/start","mcpServer":"npx -y @braintrust/mcp-server","cliInit":null,"sdkInstall":{"npm":"npm install braintrust","pip":"pip install braintrust"},"installCommand":"npm install braintrust","activeModelOrSku":"braintrust-pro","canonicalSnippet":"import { Eval } from \"braintrust\";\nEval(\"agent-eval\", { data: () => [{ input: \"hi\", expected: \"hi\" }], task: async (x) => x, scores: [] });","verifyCommand":"curl -sS https://api.braintrust.dev/v1/project -H \"Authorization: Bearer $BRAINTRUST_API_KEY\""},"quickstartSpec":{"installCommand":"npm install braintrust","envVars":["BRAINTRUST_API_KEY"],"activeModelOrSku":"braintrust-pro","canonicalSnippet":"import { Eval } from \"braintrust\";\nEval(\"agent-eval\", { data: () => [{ input: \"hi\", expected: \"hi\" }], task: async (x) => x, scores: [] });","sdkInstall":{"npm":"npm install braintrust","pip":"pip install braintrust"},"verifyCommand":"curl -sS https://api.braintrust.dev/v1/project -H \"Authorization: Bearer $BRAINTRUST_API_KEY\"","apiKeyUrl":"https://www.braintrust.dev/app/settings?subroute=api-keys","docsUrl":"https://www.braintrust.dev/docs/start","mcpServer":"npx -y @braintrust/mcp-server","bootstrapReason":null},"operationalGotchas":[{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Broader tracing competitors exist","sourceDate":"2026-10-28"},{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Price entry higher than hobby tools","sourceDate":"2026-10-28"}],"relatedPacks":[],"skipTopPick":false,"skipReason":null,"matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","constraintBundle":{"schemaVersion":"agentdepot.constraints.v1","task":null,"jtbd":"llm-as-judge-calibration","constraintsRaw":"","constraintTokens":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"typedConstraints":[],"quotaExhausted":[],"degraded":[],"healthy":[],"budgetUsd":null,"maxLatencyMs":null,"region":null,"privacy":null,"exit":[],"existing":[],"existingSubscriptions":[],"requiredCapabilities":[],"switchingCost":null},"pricingSummary":"Eval-first platform; data-based pricing with unlimited seats on many plans; strong CI eval story.","cite":"Depot Pick for llm-as-judge-calibration: braintrust (asOf 2026-10-28). See GET /api/jtbds/llm-as-judge-calibration + GET /api/methodology.","nextActions":[{"step":1,"title":"Inspect full JTBD + embeds","href":"/api/jtbds/llm-as-judge-calibration?embed=providers","why":"Confirm pick, runners-up, and methodology notes."},{"step":2,"title":"Quote sourced SKUs","href":"/api/quote?jtbd=llm-as-judge-calibration","why":"Depot Pick SKUs plus the cheapest sourced SKU whose unit matches this job. Does not change the pick. Does not blend input and output or borrow a chat rate for a non-chat job."},{"step":3,"title":"Open provider detail","href":"/api/providers/braintrust","why":"Scores, pricingNotes, strengths/weaknesses, communitySignals (claims only)."},{"step":4,"title":"Cite methodology","href":"/api/methodology","why":"Required when publishing a Depot Pick citation."},{"step":5,"title":"Grab badge SVG","href":"/api/badge?type=depot-pick&jtbd=llm-as-judge-calibration","why":"Embed in README / agent passport."},{"step":6,"title":"Post production insight","method":"POST","href":"/api/insights","why":"Unverified until curated; never auto-mutates official scores."}],"honesty":["Do not invent scores or benches.","Insights are claims until curated/reviewed; they never silently mutate official scores.","lockInRisk higher = more lock-in (worse).","normalizedUsdPerMillionTokens is set only for a single sourced token row. Otherwise null, with a machine-readable reason. Do not coerce GPU, minute, or per-1k search units.","Runners-up are not Depot Picks.","Depot overall is editorial_weighted, not Arena Elo and not an Artificial Analysis index."],"asOf":"2026-10-28","depth":{"whenNotToUse":{"text":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","source":"editorial `whenNotTopPick` (data/jtbds.json)"},"whenNotToUseReason":null,"runnerUp":{"providerId":"patronus","name":"Patronus AI","overallForJob":7.43,"pickOverallForJob":8.32,"beatsPickOn":["privacyPosture"],"source":"first resolvable entry of the job's editorially ranked `runnersUpIds`; scores are stored job-weighted scores"},"runnerUpReason":null,"lockIn":{"pickLockInRisk":5,"runnerUpLockInRisk":5.5,"scale":"0–10 stored curated estimate; higher = more lock-in (worse)","pickExitSignals":[],"runnerUpExitSignals":[],"pickNotes":[],"selfHostFallback":[],"migration":"The pick has lower stored lock-in risk than runner-up Patronus AI (5 vs 5.5).","migrationReason":null,"derivedFrom":"stored lockInRisk scores, provider tags (oss, self-host, openai-compat, open-models), free rows that list this job, and verbatim provider notes. No migration guide was invented."}},"citation":{"text":"Agent Provider Depot (v0.13.9), Depot Pick for llm-as-judge-calibration: braintrust, scores as of 2026-10-28, retrieved 2026-10-29. https://getagentdepot.com/jtbds/llm-as-judge-calibration","markdown":"[Agent Provider Depot](https://getagentdepot.com/jtbds/llm-as-judge-calibration) (v0.13.9) — Depot Pick for `llm-as-judge-calibration`: **braintrust** (scores as of 2026-10-28, retrieved 2026-10-29)","fields":{"publisher":"Agent Provider Depot","version":"0.13.9","role":"depot_pick","jtbdId":"llm-as-judge-calibration","providerId":"braintrust","scoreAsOf":"2026-10-28","retrievedOn":"2026-10-29","page":"https://getagentdepot.com/jtbds/llm-as-judge-calibration","api":"https://getagentdepot.com/api/decide?jtbd=llm-as-judge-calibration"},"howToCite":"Cite Depot as an editorial, dated recommendation, not a benchmark: name the job, the Depot Pick (or runner-up), the score asOf date and the day you retrieved it, and link the job page. Format: “Agent Provider Depot (v{version}), Depot Pick for {jobId}: {providerId}, scores as of {scoreAsOf}, retrieved {retrievedOn}. {baseUrl}/jtbds/{jobId}”. Quote prices only with their own sourceUrl and asOf. If the pick's score is stale (freshness.pick.score.stale), say so."},"freshness":{"pick":{"score":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"lastReviewed":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"sourceCheck":{"checkedAt":"2026-10-28","sourceUrls":1,"sourcesReachable":1,"pricingRows":0,"pricingRowsReconfirmed":0,"note":"A source re-check confirms the page is reachable and, for pricing rows, that the stored amount and SKU name appear on it. It does not re-derive or change the score.","reason":null}},"scoreAsOf":"2026-10-28","editorialPriorAsOf":"2026-10-28","pricingLastVerifiedAt":"2026-10-28","pickLastReviewed":"2026-10-28","runnersUpScoreAsOf":["2026-10-28","2026-10-28","2026-10-28","2026-10-28","2026-10-28"],"contractAsOf":"2026-10-03","pricingShape":"prose_notes_only","normalizedUsdPerMillionTokens":null,"normalizationStatus":"unavailable","normalizationReason":"no_sourced_price"},"pricingContract":{"normalizedUsdPerMillionTokens":null,"normalizationStatus":"unavailable","normalizationReason":"no_sourced_price","shape":"prose_notes_only","filterable":false,"priceSortEligible":false,"priceSortKeyUsdPerMillionInputTokens":null,"rows":[],"notesExcerpt":"Eval-first platform; data-based pricing with unlimited seats on many plans; strong CI eval story.","notesCheck":{"status":"no_known_contradiction","pairs":[],"review":null,"note":null},"lastReviewed":"2026-10-28","scoreAsOf":"2026-10-28","website":"https://www.braintrust.dev","doNotParse":"Do not parse pricingNotes. Use rows only. normalizedUsdPerMillionTokens is a single number only when this provider has exactly one sourced token row. Input/output, context tiers, and non-token units stay null at this level.","gap":"No sourced price row for this provider. pricingNotes stay prose. See research/gaps.md."},"links":{"jtbd":"/jtbds/llm-as-judge-calibration","jtbdApi":"/api/jtbds/llm-as-judge-calibration","leaderboard":"/api/leaderboards?jtbd=llm-as-judge-calibration","methodology":"/api/methodology","badge":"/api/badge?type=depot-pick&jtbd=llm-as-judge-calibration","decide":"/api/decide?jtbd=llm-as-judge-calibration","humanPage":"/jtbds/llm-as-judge-calibration","alternatives":"/api/alternatives?provider=braintrust","canonical":"/api/canonical","quote":"/api/quote?jtbd=llm-as-judge-calibration","falsify":"/api/falsify?jtbd=llm-as-judge-calibration"},"stackContext":{"host":null,"framework":null,"repo":null,"lang":null,"existing":[],"volume":null,"sender":null,"detectedFromTask":[]},"keep":null,"contextAdvice":null,"communityFeedback":{"pickOnThisJob":{"failed":0,"degraded":0,"worked":0,"insights":0,"withVerifiedReceipt":0,"failureModes":{},"latestAt":null,"note":"Agent-reported claims (unverified unless a receipt is verified by Depot's re-check). They never change scores or picks."},"outcomes":{"job":{"n":0,"worked":0,"degraded":0,"failed":0,"failureModes":{},"verifiedReports":0,"contributors":0,"excluded":{"seed_editorial":0,"test":0,"notFeedback":0},"reason":"no_submitted_outcome_reports_yet"},"pick":{"providerId":"braintrust","n":0,"worked":0,"degraded":0,"failed":0,"failureModes":{},"evidenceTiers":{"agent_claim":0,"self_declared":0,"reference_checked":0,"outcome_verified":0},"reason":"no_submitted_outcome_reports_yet"},"scoresMutated":false,"more":"/api/outcomes?jtbd=llm-as-judge-calibration"},"report":"POST /api/feedback {agentIdentity, outcome: failed|degraded|worked, detail, providerId, jtbdId, failureModeId?, receipt?}"}}