{"id":"llm-as-judge-calibration","slug":"llm-as-judge-calibration","name":"LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation","shortName":"LLM-as-judge calibration","problemStatement":"Calibrate LLM-as-judge graders against human gold labels and deterministic assertions so your eval harness does not reward verbose, sycophantic, or self-preferring outputs.","whenNotTopPick":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","categoryIds":["cat-observability"],"depotPickProviderId":"braintrust","depotPickRationale":"Braintrust lets teams version code-based and LLM-judge scorers side-by-side, measure judge agreement against human review queues on the same dataset rows, and swap judge models (e.g. grading an OpenAI agent with Anthropic or vice versa) in one config line.","runnersUpIds":["patronus","ragas","galileo","confident-ai","promptfoo"],"weights":{"capabilityFit":0.28,"pricePerformance":0.14,"latencyReliability":0.14,"developerDx":0.16,"privacyPosture":0.12,"lockInRisk":0.06,"maturity":0.1},"methodologyNotes":"Four mandatory calibration rules: (1) always use deterministic graders (unit tests, JSON schema, SQL/DOM state, regex) first; (2) use a cross-family judge model to eliminate self-preference bias; (3) randomize A/B pairwise order to cancel position bias and penalize verbosity; (4) require ≥85% Cohen's κ / agreement on a 50–100 row human-labeled gold slice before trusting an automated judge in CI.","tags":["evals","llm-as-judge","calibration","rubric","hallucination","quality"],"scoreMargin":0.89,"isNearTie":false,"pickConfidenceBand":"clear_leader","nearTie":{"isNearTie":false,"scoreMargin":0.89,"threshold":0.35,"pickConfidenceBand":"clear_leader","coPickProviderId":"patronus","coPickProviderName":"Patronus AI","coPickScore":7.43,"decidingFactor":"pricePerformance (7.5 vs 5.5)","methodologyWarning":null,"tieBreakerRule":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo."},"evidenceStatus":{"decisionRole":"independent_prior_and_candidate_generator","positioning":"The independent, agent-readable prior for choosing AI infrastructure — not a universal router, generic marketplace, or synthetic leaderboard.","freshnessThresholdDays":30,"freshness_policy":{"policy_version":"agentdepot.freshness_policy.v1","threshold_days":30},"freshness_state":"fresh","editorial_freshness_state":"fresh","unqualified_current_pick_allowed":true,"pick_qualification":"qualified_current_pick","promotionClass":"verified_outcome_backed","evidenceClass":"editorial_plus_verified_receipt","staleDowngraded":false,"effectiveConfidence":"high_verified","confidenceHalfWidth":0.35,"pickScoreAgeDays":0,"editorialJudgmentAgeDays":9,"pricingAgeDays":null,"isPickScoreStale":false,"isPricingStale":true,"editorialJudgmentAsOf":"2026-10-19","pricingVerifiedAt":null,"outcomeVerifiedAt":"2026-10-28","effectiveVerifiedAsOf":"2026-10-28","downgradeReasons":[],"stalenessWarning":"Editorial judgment and deterministic pricing verification are both within the 30-day freshness threshold.","promotionGate":{"deterministicPricingCheck":false,"approvedScoreReaffirm":true,"pendingHumanReviewBlock":false,"zeroUnverifiedSkus":true,"costEfficiencyConsistent":true,"lockInAndReliabilityConsistent":true,"secondModelOrRuleAudit":true,"countersignReceiptIds":["cs-54379c97","cs-6eab5b5d","cs-316f2587","cs-0e63df18","cs-1035466f"],"otsAttestationHead":"b32634d5784cbfe663be0a16cce4f60a3ff1c3d4490437be00361cb4ae5d5d59","sourceCheckDate":"2026-10-27","activePricingRowsCount":0,"freeResourcesCount":0,"runnerUpPricingRowsCount":1},"claimProvenance":{"claim_id":"claim:llm-as-judge-calibration:braintrust:depot-pick","provider_id":"braintrust","jtbd_id":"llm-as-judge-calibration","primary_url":"https://www.braintrust.dev/docs","source_type":"primary_docs","source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.braintrust.dev","final_url":"https://www.braintrust.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:fdb47a7e8e944079","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/docs","final_url":"https://www.braintrust.dev/docs","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_docs","is_claim_specific":true,"source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/pricing","final_url":"https://www.braintrust.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:60bb11c5a2c023e3","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"verifiedOutcomeSummary":{"totalVerifiedReceipts":0,"byEvidenceTier":{"E0_ASSERTION":0,"E1_SOURCE_BOUND":0,"E2_DELIVERY":0,"E3_INDEPENDENT_OUTCOME":0,"E4_REPLICATED_OUTCOME":0},"bySubjectLayer":{"provider":0,"product":0,"model_version":0,"runtime_route":0}}},"pickScoreDetails":{"scoreRange":[7.97,8.67],"formattedScore":"8.3 ± 0.35 (8.0–8.7 · editorial_plus_verified_receipt)","evidenceClass":"editorial_plus_verified_receipt","promotionClass":"verified_outcome_backed","staleDowngraded":false,"freshState":"fresh","freshness_state":"fresh","effectiveConfidence":"high_verified","confidenceInterval":{"low":7.97,"mid":8.32,"high":8.67,"halfWidth":0.35,"confidenceLevel":0.8,"basis":"narrowed_verified_outcome_interval_0.35_backed_by_countersign_receipts"},"editorialJudgmentAsOf":"2026-10-28","sourceAgeDays":9,"pricingVerifiedAt":null,"pricingAgeDays":null,"outcomeVerifiedAt":"2026-10-28","effectiveVerifiedAsOf":"2026-10-28","dissentingEvidence":["Broader tracing competitors exist","Price entry higher than hobby tools","Learning curve for eval design","When not to use top pick: Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","Runner-up Patronus AI (patronus) is the primary alternative when constraints disqualify Braintrust."],"whatWouldChangePick":["Caller constraint matching: Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","Live quota or rate-limit exhaustion on braintrust (pass ?exhausted=braintrust or liveQuota.exhaustedProviders=[\"braintrust\"] to promote patronus).","Active subscription or enterprise commit on patronus combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.braintrust.dev or unverified SKU drift detected by npm run check:sources."],"claimProvenance":{"claim_id":"claim:llm-as-judge-calibration:braintrust:depot-pick","provider_id":"braintrust","jtbd_id":"llm-as-judge-calibration","primary_url":"https://www.braintrust.dev/docs","source_type":"primary_docs","source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.braintrust.dev","final_url":"https://www.braintrust.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:fdb47a7e8e944079","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/docs","final_url":"https://www.braintrust.dev/docs","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_docs","is_claim_specific":true,"source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/pricing","final_url":"https://www.braintrust.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:60bb11c5a2c023e3","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"calibratedScore":{"point":8.32,"band":[7.97,8.67],"interval":{"lower":7.97,"upper":8.67,"half_width":0.35},"score_kind":"experimental_calibrated_band","evidence_class":"experimental","confidence":"low","confidence_ceiling":"high","age_days":0,"dissent":["Broader tracing competitors exist","Price entry higher than hobby tools","Learning curve for eval design","When not to use top pick: Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","Runner-up Patronus AI (patronus) is the primary alternative when constraints disqualify Braintrust."],"what_would_change_this":["Caller constraint matching: Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","Live quota or rate-limit exhaustion on braintrust (pass ?exhausted=braintrust or liveQuota.exhaustedProviders=[\"braintrust\"] to promote patronus).","Active subscription or enterprise commit on patronus combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.braintrust.dev or unverified SKU drift detected by npm run check:sources."],"subject_layer":"provider"},"subject_ref":{"layer":"provider","provider_id":"braintrust","product_id":"braintrust-core","model_id":null,"version":null,"route_class":null,"inheritance_policy":"no_cross_layer_inheritance"}},"runtimeRoutes":[{"routeId":"llm-as-judge-calibration:braintrust:primary","roleForJob":"depot_pick","provider":{"id":"braintrust","slug":"braintrust","name":"Braintrust","website":"https://www.braintrust.dev","status":"active"},"model":{"representativeModelId":"Braintrust Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.braintrust.dev","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.braintrust.dev (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Braintrust Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"Starter: Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - ","subscriptionFirstEligible":true,"documentedPlansCount":2},"regions":{"supportedRegions":["us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.6,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"patronus"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:managed_cloud_service:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:direct_api:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"direct_api","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"llm-as-judge-calibration:patronus:primary","roleForJob":"runner_up","provider":{"id":"patronus","slug":"patronus","name":"Patronus AI","website":"https://www.patronus.ai","status":"active"},"model":{"representativeModelId":"Patronus AI Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.patronus.ai","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.patronus.ai (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Patronus AI Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=patronus","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global","us"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":false,"parallelToolCalls":false,"mcpServerAvailable":false,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.5,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"braintrust"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:patronus:patronus-primary:managed_cloud_service:global","provider_id":"patronus","product_id":"patronus-core","model_id":"patronus-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:patronus:patronus-primary:direct_api:global","provider_id":"patronus","product_id":"patronus-core","model_id":"patronus-primary","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"llm-as-judge-calibration:ragas:primary","roleForJob":"runner_up","provider":{"id":"ragas","slug":"ragas","name":"Ragas","website":"https://docs.ragas.io","status":"active"},"model":{"representativeModelId":"Ragas Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://docs.ragas.io","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://docs.ragas.io (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Ragas Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":true,"freeTierSummary":"Ragas (oss, asOf 2026-10-25)","rateLimitsSummary":"See /api/limits?provider=ragas","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global","us"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":false,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":7.5,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":9.2,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"braintrust"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:ragas:ragas-primary:self_hosted_or_oss_compatible:global","provider_id":"ragas","product_id":"ragas-core","model_id":"ragas-primary","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"global","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:ragas:ragas-primary:direct_api:global","provider_id":"ragas","product_id":"ragas-core","model_id":"ragas-primary","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"llm-as-judge-calibration:galileo:primary","roleForJob":"runner_up","provider":{"id":"galileo","slug":"galileo","name":"Galileo","website":"https://www.galileo.ai","status":"active"},"model":{"representativeModelId":"Pro","skuId":"galileo-pro","unit":"usd_per_month","amountUsd":100,"sourceUrl":"https://galileo.ai/pricing","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.galileo.ai (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Galileo Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=galileo","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global","us"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":false,"parallelToolCalls":false,"mcpServerAvailable":false,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":2,"unverifiedSkus":0,"liveFailoverTargetId":"braintrust"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:galileo:Pro:managed_cloud_service:global","provider_id":"galileo","product_id":"galileo-core","model_id":"Pro","version":"2026-10-28","route_class":"managed_cloud_service","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:galileo:Pro:direct_api:global","provider_id":"galileo","product_id":"galileo-core","model_id":"Pro","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]}],"depth":{"whenNotToUse":{"text":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","source":"editorial `whenNotTopPick` (data/jtbds.json)"},"whenNotToUseReason":null,"runnerUp":{"providerId":"patronus","name":"Patronus AI","overallForJob":7.43,"pickOverallForJob":8.32,"beatsPickOn":["privacyPosture"],"source":"first resolvable entry of the job's editorially ranked `runnersUpIds`; scores are stored job-weighted scores"},"runnerUpReason":null,"lockIn":{"pickLockInRisk":5,"runnerUpLockInRisk":5.5,"scale":"0–10 stored curated estimate; higher = more lock-in (worse)","pickExitSignals":[],"runnerUpExitSignals":[],"pickNotes":[],"selfHostFallback":[],"migration":"The pick has lower stored lock-in risk than runner-up Patronus AI (5 vs 5.5).","migrationReason":null,"derivedFrom":"stored lockInRisk scores, provider tags (oss, self-host, openai-compat, open-models), free rows that list this job, and verbatim provider notes. No migration guide was invented."}},"freshness":{"pick":{"score":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"lastReviewed":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"sourceCheck":{"checkedAt":"2026-10-28","sourceUrls":1,"sourcesReachable":1,"pricingRows":0,"pricingRowsReconfirmed":0,"note":"A source re-check confirms the page is reachable and, for pricing rows, that the stored amount and SKU name appear on it. It does not re-derive or change the score.","reason":null}}},"whenNotTopPickRules":[{"ifConstraint":"escape_hatch_clause_1","recommendedProviderId":"patronus","activeModelOrSku":"patronus-standard","reason":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI"},{"ifConstraint":"escape_hatch_clause_2","recommendedProviderId":"ragas","activeModelOrSku":"ragas-oss","reason":"research-grade RAG faithfulness/context metrics — Ragas"},{"ifConstraint":"escape_hatch_clause_3","recommendedProviderId":"galileo","activeModelOrSku":"galileo-evaluate","reason":"enterprise guardrail + hallucination scoring — Galileo"},{"ifConstraint":"escape_hatch_clause_4","recommendedProviderId":"confident-ai","activeModelOrSku":"confident-ai-standard","reason":"pytest-native G-Eval/DAG rubrics — Confident AI"},{"ifConstraint":"exit:self_host_or_open_weights","recommendedProviderId":"promptfoo","activeModelOrSku":"promptfoo-oss","reason":"OSS local assertion matrices — Promptfoo."}],"bootstrap":{"authType":"api_key","envVars":["BRAINTRUST_API_KEY"],"signupUrl":"https://www.braintrust.dev/signup","apiKeyUrl":"https://www.braintrust.dev/app/settings?subroute=api-keys","docsUrl":"https://www.braintrust.dev/docs/start","mcpServer":"npx -y @braintrust/mcp-server","cliInit":null,"sdkInstall":{"npm":"npm install braintrust","pip":"pip install braintrust"},"installCommand":"npm install braintrust","activeModelOrSku":"braintrust-pro","canonicalSnippet":"import { Eval } from \"braintrust\";\nEval(\"agent-eval\", { data: () => [{ input: \"hi\", expected: \"hi\" }], task: async (x) => x, scores: [] });","verifyCommand":"curl -sS https://api.braintrust.dev/v1/project -H \"Authorization: Bearer $BRAINTRUST_API_KEY\""},"quickstartSpec":{"installCommand":"npm install braintrust","envVars":["BRAINTRUST_API_KEY"],"activeModelOrSku":"braintrust-pro","canonicalSnippet":"import { Eval } from \"braintrust\";\nEval(\"agent-eval\", { data: () => [{ input: \"hi\", expected: \"hi\" }], task: async (x) => x, scores: [] });","sdkInstall":{"npm":"npm install braintrust","pip":"pip install braintrust"},"verifyCommand":"curl -sS https://api.braintrust.dev/v1/project -H \"Authorization: Bearer $BRAINTRUST_API_KEY\"","apiKeyUrl":"https://www.braintrust.dev/app/settings?subroute=api-keys","docsUrl":"https://www.braintrust.dev/docs/start","mcpServer":"npx -y @braintrust/mcp-server","bootstrapReason":null},"operationalGotchas":[{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Broader tracing competitors exist","sourceDate":"2026-10-28"},{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Price entry higher than hobby tools","sourceDate":"2026-10-28"}],"relatedPacks":[],"depotPick":{"id":"braintrust","slug":"braintrust","name":"Braintrust","tagline":"Depot pick when evals are the product quality gate. Pair with Helicone/Langfuse for cheap request logs if…","overall":8.32},"runnersUp":[{"id":"patronus","slug":"patronus","name":"Patronus AI","tagline":"Layer on when regulated hallucination/safety gates matter more than general CI evals. Lakera for injection…","overall":7.43},{"id":"ragas","slug":"ragas","name":"Ragas","tagline":"Default RAG metric library. Pair with Promptfoo/Braintrust for broader agent evals.","overall":8.3},{"id":"galileo","slug":"galileo","name":"Galileo","tagline":"Pick for production hallucination measurement + guardrails. Braintrust/Promptfoo for lighter eval loops.","overall":7.57},{"id":"confident-ai","slug":"confident-ai","name":"Confident AI","tagline":"Best when your team already thinks in pytest/DeepEval. Braintrust remains Depot hosted-evals default…","overall":8.05},{"id":"promptfoo","slug":"promptfoo","name":"Promptfoo","tagline":"Default OSS eval harness. Braintrust when you want hosted collaboration; Promptfoo when you want local/CI…","overall":8.41}]}