{"jtbd":{"id":"production-guardrail-evals","slug":"production-guardrail-evals","name":"Production guardrails & hallucination monitoring","shortName":"Production guardrails & hallucination mo…","problemStatement":"Detect and block bad outputs in production with research-grade metrics."},"pick":{"id":"galileo","slug":"galileo","name":"Galileo","tagline":"Pick for production hallucination measurement + guardrails. Braintrust/Promptfoo for lighter eval loops.","website":"https://www.galileo.ai","overall":7.51,"scoreBand":{"low":6.9,"mid":7.5,"high":8.1,"halfWidth":0.6,"basis":"editorial_prior","display":"6.9–8.1 (editorial prior)"},"scoreLayers":{"editorialPrior":7.51,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","pricingNotes":"As of 2026-09-14 (galileo.ai/pricing live): Free **$0/mo** — 5,000 traces/mo, unlimited users/custom evals. Pro **$100/mo** (annual; scales with traces) — 50,000 traces/mo, standard RBAC, advanced analytics, Slack support. Enterprise contact — unlimited traces, VPC/on-prem, SSO, real-time guardrails, 24/7.","scores":{"capabilityFit":8.4,"pricePerformance":6.2,"latencyReliability":8,"developerDx":7.8,"privacyPosture":8,"lockInRisk":6,"maturity":7.8,"overall":7.51,"kind":"curated_estimate","confidence":"high","asOf":"2026-09-14"}},"runnersUp":[{"id":"braintrust","slug":"braintrust","name":"Braintrust","tagline":"Depot pick when evals are the product quality gate. Pair with Helicone/Langfuse for cheap request logs if…","website":"https://www.braintrust.dev","overall":8.28,"scoreBand":{"low":7.7,"mid":8.3,"high":8.9,"halfWidth":0.6,"basis":"editorial_prior","display":"7.7–8.9 (editorial prior)"},"scoreLayers":{"editorialPrior":8.28,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-10-19","pricingNotes":"Eval-first platform; data-based pricing with unlimited seats on many plans; strong CI eval story.","scores":{"capabilityFit":9.2,"pricePerformance":7.5,"latencyReliability":8.6,"developerDx":8.8,"privacyPosture":8,"lockInRisk":5,"maturity":8.2,"overall":8.28,"kind":"curated_estimate","confidence":"high","asOf":"2026-10-19"}},{"id":"maxim","slug":"maxim","name":"Maxim AI","tagline":"Pick Maxim when pre-release agent simulation is the pain and seats are few. Braintrust for general evals…","website":"https://www.getmaxim.ai","overall":7.53,"scoreBand":{"low":6.9,"mid":7.5,"high":8.1,"halfWidth":0.6,"basis":"editorial_prior","display":"6.9–8.1 (editorial prior)"},"scoreLayers":{"editorialPrior":7.53,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","pricingNotes":"As of 2026-09-14 (official getmaxim.ai / webflow pricing): Developer Free (≤3 seats, 10k logs/mo, 3-day retention); Professional $29/seat/mo (100k logs, simulation + online evals, 7-day retention); Business $49/seat/mo (500k logs, 30-day retention, RBAC, scheduled runs); Enterprise custom (SSO, in-VPC, SOC2/HIPAA/GDPR, BAA).","scores":{"capabilityFit":8,"pricePerformance":7.2,"latencyReliability":7.8,"developerDx":8,"privacyPosture":8.2,"lockInRisk":5.5,"maturity":6.8,"overall":7.53,"kind":"curated_estimate","confidence":"high","asOf":"2026-09-14"}},{"id":"langsmith","slug":"langsmith","name":"LangSmith","tagline":"If you live in LangGraph, LangSmith is the obvious observatory. Otherwise compare Langfuse/Helicone first.","website":"https://www.langchain.com/langsmith","overall":8.14,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.14,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","pricingNotes":"Seat + trace volume pricing; deep LangChain/LangGraph integration; Plus tiers from tens of $/mo.","scores":{"capabilityFit":9,"pricePerformance":7,"latencyReliability":8.8,"developerDx":9,"privacyPosture":7.8,"lockInRisk":6.5,"maturity":8.8,"overall":8.14,"kind":"curated_estimate","confidence":"high","asOf":"2026-09-14"}},{"id":"arthur-ai","slug":"arthur-ai","name":"Arthur AI","tagline":"Monitoring/evals peer with a real $60 Premium. Prefer Braintrust/Langfuse for tracing defaults; Arthur when…","website":"https://www.arthur.ai","overall":7.87,"scoreBand":{"low":7.3,"mid":7.9,"high":8.5,"halfWidth":0.6,"basis":"editorial_prior","display":"7.3–8.5 (editorial prior)"},"scoreLayers":{"editorialPrior":7.87,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","pricingNotes":"As of 2026-09-14 (arthur.ai/pricing): Free $0 — 4 use cases, 5k jobs / 300k spans / 12k inferences / 3k evals, 7-day retention. Premium $60/mo — 100 use cases, 20k jobs / 1.2M spans / 100k inferences / 75k evals, 30-day retention, custom dashboards/alerts. Enterprise custom (VPC, SSO, BAA, SLA). Arthur Shield LLM firewall is separately sales-gated.","scores":{"capabilityFit":8.2,"pricePerformance":8,"latencyReliability":8,"developerDx":8.2,"privacyPosture":7.8,"lockInRisk":4.5,"maturity":8,"overall":7.87,"kind":"curated_estimate","confidence":"high","asOf":"2026-09-14"}},{"id":"patronus","slug":"patronus","name":"Patronus AI","tagline":"Layer on when regulated hallucination/safety gates matter more than general CI evals. Lakera for injection…","website":"https://www.patronus.ai","overall":7.37,"scoreBand":{"low":6.8,"mid":7.4,"high":8,"halfWidth":0.6,"basis":"editorial_prior","display":"6.8–8.0 (editorial prior)"},"scoreLayers":{"editorialPrior":7.37,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","pricingNotes":"As of 2026-09-14: Enterprise evaluation/safety platform (Lynx hallucination detector, Glider judge). Public list price thin — secondary reports cite ~$25/mo Base + per-1k evaluator calls; treat as sales-led until official card pinned. Series-B backed.","scores":{"capabilityFit":8.4,"pricePerformance":5.5,"latencyReliability":8,"developerDx":7.2,"privacyPosture":8.5,"lockInRisk":5.5,"maturity":7.5,"overall":7.37,"kind":"curated_estimate","confidence":"medium","asOf":"2026-09-14"}},{"id":"confident-ai","slug":"confident-ai","name":"Confident AI","tagline":"Best when your team already thinks in pytest/DeepEval. Braintrust remains Depot hosted-evals default…","website":"https://www.confident-ai.com","overall":8.06,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.06,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","pricingNotes":"As of 2026-09-14 (confident-ai.com/pricing): Free forever (2 seats, 1 project, 5 test runs/week, 1 GB-mo traces). Starter $200/mo (5 projects, 5 GB-mo then $1/GB-mo). Team $2,000/mo (unlimited projects, 75 GB-mo). Enterprise custom (on-prem, HIPAA, residency). Online eval tokens ~$0.05/$0.40 per 1M in/out.","scores":{"capabilityFit":8.6,"pricePerformance":7.8,"latencyReliability":8,"developerDx":9,"privacyPosture":7.8,"lockInRisk":4.5,"maturity":7.2,"overall":8.06,"kind":"curated_estimate","confidence":"high","asOf":"2026-09-14"}},{"id":"honeyhive","slug":"honeyhive","name":"HoneyHive","tagline":"Use the free Developer tier to stand up agent evals/traces in a day; Enterprise when you need self-host…","website":"https://www.honeyhive.ai","overall":8.08,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.08,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","pricingNotes":"As of 2026-09-14 (honeyhive.ai/pricing): Developer free (no card) — 10k events/mo, 5 users, 1 workspace, 30-day retention, 1k RPM, evals + prompt versioning + CI. Enterprise custom (unlimited users/workspaces, SAML, custom retention, SLA, DPA/BAA, multi-tenant / single-tenant / hybrid / self-host). Data residency default AWS us-west-2. SOC2 Type II / GDPR / HIPAA. Startup discount < $5M raised.","scores":{"capabilityFit":8.4,"pricePerformance":7.8,"latencyReliability":8,"developerDx":8.6,"privacyPosture":8.8,"lockInRisk":4.2,"maturity":7.4,"overall":8.08,"kind":"curated_estimate","confidence":"high","asOf":"2026-09-14"}}],"scoreBand":{"low":6.9,"mid":7.5,"high":8.1,"halfWidth":0.6,"basis":"editorial_prior","display":"6.9–8.1 (editorial prior)"},"scoreLayers":{"editorialPrior":7.51,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"whyPickOverHigherScore":{"pickId":"galileo","pickName":"Galileo","pickScore":7.51,"highestScoredCandidateId":"braintrust","highestScoredCandidateName":"Braintrust","highestScore":8.28,"pickHasLowerCompositeScore":true,"delta":-0.77,"pickLeadsOnDimensions":[],"explanation":"Galileo (7.51) is the editorial pick for Production guardrails & hallucination monitoring over higher-composite Braintrust (8.28) because composite scores are provider-wide priors while Galileo wins on job-specific execution fit: Capability of guardrails + maturity."},"cheapest":null,"recommendedWorkloadQuote":{"providerId":"galileo","providerName":"Galileo","skuId":"galileo-free","item":"Free","plan":"Free — 5,000 traces per month","unit":"usd_per_month","amountUsd":0,"outputSkuId":null,"outputItem":null,"outputAmountUsd":null,"outputUnit":null,"sourceUrl":"https://galileo.ai/pricing","asOf":"2026-10-03","pricingLastVerifiedAt":"2026-10-27","isFlagshipForJob":false,"differsFromFloorCheapest":false,"representativeQuote":{"skuId":"galileo-free","item":"Free","plan":"Free — 5,000 traces per month","amountUsd":0,"inputOrUnitUsd":0,"unit":"usd_per_month","outputUsd":null,"outputUnit":null,"sourceUrl":"https://galileo.ai/pricing","asOf":"2026-10-03"},"estimatedTaskCost":{"tasks":1000,"retryAndToolOverheadMultiplier":1.15,"retryToolOverheadMultiplier":1.15,"unitsPerTask":1,"baseCostPer1kTasksUsd":0,"totalUsd":0,"estimatedTotalUsd":0,"perTaskUsd":0,"estimatedCostPerTaskUsd":0,"formula":"Monthly base plan $0/mo covering 1,000 tasks (1.15x overhead buffer)","assumptions":"Monthly subscription plan amortized over 1,000 tasks/month."}},"representativeQuote":{"skuId":"galileo-free","item":"Free","plan":"Free — 5,000 traces per month","amountUsd":0,"inputOrUnitUsd":0,"unit":"usd_per_month","outputUsd":null,"outputUnit":null,"sourceUrl":"https://galileo.ai/pricing","asOf":"2026-10-03"},"estimatedTaskCost":{"tasks":1000,"retryAndToolOverheadMultiplier":1.15,"retryToolOverheadMultiplier":1.15,"unitsPerTask":1,"baseCostPer1kTasksUsd":0,"totalUsd":0,"estimatedTotalUsd":0,"perTaskUsd":0,"estimatedCostPerTaskUsd":0,"formula":"Monthly base plan $0/mo covering 1,000 tasks (1.15x overhead buffer)","assumptions":"Monthly subscription plan amortized over 1,000 tasks/month."},"freeAlternatives":[{"id":"guardrails","name":"Guardrails AI","kind":"dev_tool","providerId":null,"jtbdIds":["production-guardrail-evals"],"whatYouGet":"Python framework. The README says input and output guards detect, quantify, and mitigate specific risks. The fetched README does not say hallucination monitoring or research-grade metrics. Hosted remote inferencing is not this row.","cost":"oss","license":"Apache-2.0","limits":null,"sourceUrl":"https://raw.githubusercontent.com/guardrails-ai/guardrails/main/LICENSE","asOf":"2026-10-25","notes":"LICENSE fetched 2026-10-03 is Apache License 2.0. README fetched the same day says hosted remote inferencing is being discontinued, planned cutoff August 25, 2026. Re-fetched twice 2026-10-25 (~10 s apart, plain curl, no login): HTTP 200, no redirect, byte-identical: 'Apache License Version 2.0, January 2004'. README.md (https://raw.githubusercontent.com/guardrails-ai/guardrails/main/README.md) re-fetched twice 2026-10-25 (~10 s apart, plain curl, no login): HTTP 200, no redirect, byte-identical; it still says Guardrails is discontinuing its hosted remote inferencing, 'Planned cutoff: August 25, 2026.' (cutoff unchanged). License unchanged.","freeType":"dev_tool","substitutable":true}],"referenceDocs":[],"falsification":{"jtbdId":"production-guardrail-evals","jtbdName":"Production guardrails & hallucination monitoring","currentPickId":"galileo","currentPickScore":7.51,"challengerId":"braintrust","challengerScore":8.28,"currentScoreGap":0.77,"minVerifiedOutcomes":10,"requiredOutcomeMargin":0.92,"requiredSuccessRateDelta":0.15,"maxPickPriceIncreasePct":25,"whatWouldChangePick":["Braintrust (braintrust) outperforms Galileo (galileo) by >= 0.92 outcome points (or >= 15% worked-rate delta) across >= 10 Countersign-verified production outcomes.","A verified price increase > 25% on Galileo's representative SKU for production-guardrail-evals without a corresponding capability gain.","Live caller rail exhaustion (?exhausted=galileo) or sustained >= 30% degraded/failed rate in verified production outcomes."],"frozenHoldoutPolicy":{"holdoutSplit":"20% frozen benchmark holdout tasks + 80% live production outcome receipts","requireCountersignReceipt":true,"requiredEvidenceTiers":["outcome_verified","reference_checked"],"minDistinctContributors":2,"antiGamingRule":"Unverified self-reports and seed/editorial rows never mutate picks or scores."},"triggers":[{"kind":"verified_outcomes_outperformance","challengerId":"braintrust","minVerifiedOutcomes":10,"requiredOutcomeMargin":0.92,"requiredWorkedRateDelta":0.15},{"kind":"verified_price_increase","providerId":"galileo","maxPriceIncreasePct":25},{"kind":"live_rail_exhaustion_or_degradation","providerId":"galileo","immediateOverrideUrl":"/api/decide?jtbd=production-guardrail-evals&exhausted=galileo"}],"submissionSpec":{"falsifyEndpoint":"/api/falsify?jtbd=production-guardrail-evals","outcomesEndpoint":"POST /api/feedback","correctionsEndpoint":"POST /api/corrections","payloadTemplate":{"agentIdentity":"agent:<your-agent-id>","jtbdId":"production-guardrail-evals","providerId":"braintrust","outcome":"worked","detail":"Comparative production evaluation on production-guardrail-evals (n>=10 tasks) with Countersign receipt.","confidence":"high","receipt":{"countersign":{"claim_id":"cs-00000000","status":"verified","evidence_tier":"outcome_verified","counts_as_verified":true}}}}},"whatWouldChangePick":["Braintrust (braintrust) outperforms Galileo (galileo) by >= 0.92 outcome points (or >= 15% worked-rate delta) across >= 10 Countersign-verified production outcomes.","A verified price increase > 25% on Galileo's representative SKU for production-guardrail-evals without a corresponding capability gain.","Live caller rail exhaustion (?exhausted=galileo) or sustained >= 30% degraded/failed rate in verified production outcomes."],"routePlan":{"jtbdId":"production-guardrail-evals","primary":{"providerId":"galileo","name":"Galileo","overall":7.51,"scoreBand":{"low":6.9,"mid":7.5,"high":8.1,"halfWidth":0.6,"basis":"editorial_prior","display":"6.9–8.1 (editorial prior)"},"scoreLayers":{"editorialPrior":7.51,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","role":"primary","reason":"Editorial Depot Pick for Production guardrails & hallucination monitoring; healthy/eligible under caller constraints."},"fallback":[{"providerId":"braintrust","name":"Braintrust","overall":8.28,"scoreBand":{"low":7.7,"mid":8.3,"high":8.9,"halfWidth":0.6,"basis":"editorial_prior","display":"7.7–8.9 (editorial prior)"},"scoreLayers":{"editorialPrior":8.28,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-10-19","role":"fallback","reason":"Ranked fallback route for Production guardrails & hallucination monitoring."},{"providerId":"maxim","name":"Maxim AI","overall":7.53,"scoreBand":{"low":6.9,"mid":7.5,"high":8.1,"halfWidth":0.6,"basis":"editorial_prior","display":"6.9–8.1 (editorial prior)"},"scoreLayers":{"editorialPrior":7.53,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","role":"fallback","reason":"Ranked fallback route for Production guardrails & hallucination monitoring."},{"providerId":"langsmith","name":"LangSmith","overall":8.14,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.14,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","role":"fallback","reason":"Ranked fallback route for Production guardrails & hallucination monitoring."},{"providerId":"arthur-ai","name":"Arthur AI","overall":7.87,"scoreBand":{"low":7.3,"mid":7.9,"high":8.5,"halfWidth":0.6,"basis":"editorial_prior","display":"7.3–8.5 (editorial prior)"},"scoreLayers":{"editorialPrior":7.87,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","role":"fallback","reason":"Ranked fallback route for Production guardrails & hallucination monitoring."},{"providerId":"patronus","name":"Patronus AI","overall":7.37,"scoreBand":{"low":6.8,"mid":7.4,"high":8,"halfWidth":0.6,"basis":"editorial_prior","display":"6.8–8.0 (editorial prior)"},"scoreLayers":{"editorialPrior":7.37,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","role":"fallback","reason":"Ranked fallback route for Production guardrails & hallucination monitoring."},{"providerId":"confident-ai","name":"Confident AI","overall":8.06,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.06,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","role":"fallback","reason":"Ranked fallback route for Production guardrails & hallucination monitoring."},{"providerId":"honeyhive","name":"HoneyHive","overall":8.08,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.08,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","role":"fallback","reason":"Ranked fallback route for Production guardrails & hallucination monitoring."}],"disqualified":[],"expectedRegret":{"scoreDeltaVsUnconstrainedPick":0,"unconstrainedPickId":"galileo","activePrimaryId":"galileo","summary":"0.00 score regret: unconstrained editorial pick satisfies all caller constraints."},"evidenceAgeDays":{"editorialScoreAgeDays":43,"pricingVerifiedAgeDays":0,"editorialPriorAsOf":"2026-09-14","pricingLastVerifiedAt":"2026-10-27"},"confidenceBand":{"low":6.9,"mid":7.5,"high":8.1,"halfWidth":0.6,"basis":"editorial_prior","display":"6.9–8.1 (editorial prior)"},"constraintsApplied":false,"constraintBundle":{"task":null,"jtbd":"production-guardrail-evals","quotaExhausted":[],"degraded":[],"healthy":[],"budgetUsd":null,"maxLatencyMs":null,"region":null,"privacy":null,"exit":[],"existing":[]}},"pricingLastVerifiedAt":"2026-10-27","editorialPriorAsOf":"2026-09-14","tie":false,"scoreGap":-0.77,"decidingFactor":null,"tieNote":null,"rationale":"Capability of guardrails + maturity.","whenNotTopPick":"Pre-release simulation focus — Maxim; lightweight CI — Promptfoo.","skipTopPick":false,"skipReason":null,"matchedConstraints":[],"pricingSummary":"As of 2026-09-14 (galileo.ai/pricing live): Free **$0/mo** — 5,000 traces/mo, unlimited users/custom evals. Pro **$100/mo** (annual; scales with traces) — 50,000 traces/mo, standard RBAC, advanced analytics, Slack support. Enterprise contact — unlimited traces, VPC/on-prem, SSO, …","cite":"Depot Pick for production-guardrail-evals: galileo (asOf 2026-09-14). See GET /api/jtbds/production-guardrail-evals + GET /api/methodology.","nextActions":[{"step":1,"title":"Inspect full JTBD + embeds","href":"/api/jtbds/production-guardrail-evals?embed=providers","why":"Confirm pick, runners-up, and methodology notes."},{"step":2,"title":"Quote sourced SKUs","href":"/api/quote?jtbd=production-guardrail-evals","why":"Depot Pick SKUs plus the cheapest sourced SKU whose unit matches this job. Does not change the pick. Does not blend input and output or borrow a chat rate for a non-chat job."},{"step":3,"title":"Open provider detail","href":"/api/providers/galileo","why":"Scores, pricingNotes, strengths/weaknesses, communitySignals (claims only)."},{"step":4,"title":"Cite methodology","href":"/api/methodology","why":"Required when publishing a Depot Pick citation."},{"step":5,"title":"Grab badge SVG","href":"/api/badge?type=depot-pick&jtbd=production-guardrail-evals","why":"Embed in README / agent passport."},{"step":6,"title":"Post production insight","method":"POST","href":"/api/insights","why":"Unverified until curated; never auto-mutates official scores."}],"honesty":["Do not invent scores or benches.","Insights are claims until curated/reviewed; they never silently mutate official scores.","lockInRisk higher = more lock-in (worse).","normalizedUsdPerMillionTokens is set only for a single sourced token row. Otherwise null, with a machine-readable reason. Do not coerce GPU, minute, or per-1k search units.","Runners-up are not Depot Picks.","Depot overall is editorial_weighted, not Arena Elo and not an Artificial Analysis index."],"asOf":"2026-09-14","depth":{"whenNotToUse":{"text":"Pre-release simulation focus — Maxim; lightweight CI — Promptfoo.","source":"editorial `whenNotTopPick` (data/jtbds.json)"},"whenNotToUseReason":null,"runnerUp":{"providerId":"braintrust","name":"Braintrust","overallForJob":8.28,"pickOverallForJob":7.51,"beatsPickOn":["capabilityFit","pricePerformance","latencyReliability","developerDx","lockInRisk","maturity"],"source":"first resolvable entry of the job's editorially ranked `runnersUpIds`; scores are stored job-weighted scores"},"runnerUpReason":null,"lockIn":{"pickLockInRisk":6,"runnerUpLockInRisk":5,"scale":"0–10 stored curated estimate; higher = more lock-in (worse)","pickExitSignals":[],"runnerUpExitSignals":[],"pickNotes":[],"selfHostFallback":[{"id":"guardrails","name":"Guardrails AI","license":"Apache-2.0","sourceUrl":"https://raw.githubusercontent.com/guardrails-ai/guardrails/main/LICENSE"}],"migration":"Runner-up Braintrust has lower stored lock-in risk (5 vs 6, 0–10, higher = more lock-in). Self-host fallback listed for this job: Guardrails AI.","migrationReason":null,"derivedFrom":"stored lockInRisk scores, provider tags (oss, self-host, openai-compat, open-models), free rows that list this job, and verbatim provider notes. No migration guide was invented."}},"citation":{"text":"Agent Provider Depot (v0.12.149), Depot Pick for production-guardrail-evals: galileo, scores as of 2026-09-14, retrieved 2026-10-27. https://getagentdepot.com/jtbds/production-guardrail-evals","markdown":"[Agent Provider Depot](https://getagentdepot.com/jtbds/production-guardrail-evals) (v0.12.149) — Depot Pick for `production-guardrail-evals`: **galileo** (scores as of 2026-09-14, retrieved 2026-10-27)","fields":{"publisher":"Agent Provider Depot","version":"0.12.149","role":"depot_pick","jtbdId":"production-guardrail-evals","providerId":"galileo","scoreAsOf":"2026-09-14","retrievedOn":"2026-10-27","page":"https://getagentdepot.com/jtbds/production-guardrail-evals","api":"https://getagentdepot.com/api/decide?jtbd=production-guardrail-evals"},"howToCite":"Cite Depot as an editorial, dated recommendation, not a benchmark: name the job, the Depot Pick (or runner-up), the score asOf date and the day you retrieved it, and link the job page. Format: “Agent Provider Depot (v{version}), Depot Pick for {jobId}: {providerId}, scores as of {scoreAsOf}, retrieved {retrievedOn}. {baseUrl}/jtbds/{jobId}”. Quote prices only with their own sourceUrl and asOf. If the pick's score is stale (freshness.pick.score.stale), say so."},"freshness":{"pick":{"score":{"asOf":"2026-09-14","ageDays":43,"stale":true,"staleAfterDays":30,"reason":null},"lastReviewed":{"asOf":"2026-09-14","ageDays":43,"stale":true,"staleAfterDays":30,"reason":null},"sourceCheck":{"checkedAt":"2026-10-27","sourceUrls":3,"sourcesReachable":3,"pricingRows":2,"pricingRowsReconfirmed":2,"note":"A source re-check confirms the page is reachable and, for pricing rows, that the stored amount and SKU name appear on it. It does not re-derive or change the score.","reason":null}},"scoreAsOf":"2026-09-14","editorialPriorAsOf":"2026-09-14","pricingLastVerifiedAt":"2026-10-27","pickLastReviewed":"2026-09-14","runnersUpScoreAsOf":["2026-10-19","2026-09-14","2026-09-14","2026-09-14","2026-09-14","2026-09-14","2026-09-14"],"contractAsOf":"2026-10-03","pricingShape":"sourced_rows","normalizedUsdPerMillionTokens":null,"normalizationStatus":"unit_mismatch","normalizationReason":"unit_mismatch"},"pricingContract":{"normalizedUsdPerMillionTokens":null,"normalizationStatus":"unit_mismatch","normalizationReason":"unit_mismatch","shape":"sourced_rows","filterable":false,"priceSortEligible":false,"priceSortKeyUsdPerMillionInputTokens":null,"rows":[{"id":"galileo-free","providerId":"galileo","sku":"Free","unit":"usd_per_month","amountUsd":0,"currency":"USD","sourceUrl":"https://galileo.ai/pricing","asOf":"2026-10-03","normalizationStatus":"unit_mismatch","normalizedUsdPerMillionTokens":null,"normalizationReason":"unit_mismatch_per_month","priceSortEligible":false,"qualifier":"5,000 traces per month."},{"id":"galileo-pro","providerId":"galileo","sku":"Pro","unit":"usd_per_month","amountUsd":100,"currency":"USD","sourceUrl":"https://galileo.ai/pricing","asOf":"2026-10-03","normalizationStatus":"unit_mismatch","normalizedUsdPerMillionTokens":null,"normalizationReason":"unit_mismatch_per_month","priceSortEligible":false,"qualifier":"Card shows $100/month with Billed yearly and Save 33%. Month-to-month amount is not shown and was not reconstructed. Page says pricing scales with traces."}],"notesExcerpt":"As of 2026-09-14 (galileo.ai/pricing live): Free **$0/mo** — 5,000 traces/mo, unlimited users/custom evals. Pro **$100/mo** (annual; scales with traces) — 50,000 traces/mo, standard RBAC, advanced analytics, Slack support. Enterprise contact — unlimited traces, VPC/on-prem, SSO, …","notesCheck":{"status":"no_known_contradiction","pairs":[],"review":null,"note":null},"lastReviewed":"2026-09-14","scoreAsOf":"2026-09-14","website":"https://www.galileo.ai","doNotParse":"Do not parse pricingNotes. Use rows only. normalizedUsdPerMillionTokens is a single number only when this provider has exactly one sourced token row. Input/output, context tiers, and non-token units stay null at this level.","gap":"Partial coverage. Providers without a fetched SKU are omitted. See GET /api/pricing."},"links":{"jtbd":"/jtbds/production-guardrail-evals","jtbdApi":"/api/jtbds/production-guardrail-evals","leaderboard":"/api/leaderboards?jtbd=production-guardrail-evals","methodology":"/api/methodology","badge":"/api/badge?type=depot-pick&jtbd=production-guardrail-evals","decide":"/api/decide?jtbd=production-guardrail-evals","humanPage":"/jtbds/production-guardrail-evals","alternatives":"/api/alternatives?provider=galileo","canonical":"/api/canonical","quote":"/api/quote?jtbd=production-guardrail-evals","falsify":"/api/falsify?jtbd=production-guardrail-evals"},"constraintBundle":{"task":null,"jtbd":"production-guardrail-evals","quotaExhausted":[],"degraded":[],"healthy":[],"budgetUsd":null,"maxLatencyMs":null,"region":null,"privacy":null,"exit":[],"existing":[]},"stackContext":{"host":null,"framework":null,"repo":null,"lang":null,"existing":[],"volume":null,"sender":null,"detectedFromTask":[]},"keep":null,"contextAdvice":null,"communityFeedback":{"pickOnThisJob":{"failed":0,"degraded":0,"worked":0,"insights":0,"withVerifiedReceipt":0,"failureModes":{},"latestAt":null,"note":"Agent-reported claims (unverified unless a receipt is verified by Depot's re-check). They never change scores or picks."},"outcomes":{"job":{"n":0,"worked":0,"degraded":0,"failed":0,"failureModes":{},"verifiedReports":0,"contributors":0,"excluded":{"seed_editorial":0,"test":0,"notFeedback":0},"reason":"no_submitted_outcome_reports_yet"},"pick":{"providerId":"galileo","n":0,"worked":0,"degraded":0,"failed":0,"failureModes":{},"evidenceTiers":{"agent_claim":0,"self_declared":0,"reference_checked":0,"outcome_verified":0},"reason":"no_submitted_outcome_reports_yet"},"scoresMutated":false,"more":"/api/outcomes?jtbd=production-guardrail-evals"},"report":"POST /api/feedback {agentIdentity, outcome: failed|degraded|worked, detail, providerId, jtbdId, failureModeId?, receipt?}"}}