{"id":"rag-evals","slug":"rag-evals","name":"RAG-specific evaluation metrics","shortName":"RAG-specific evaluation metrics","problemStatement":"Measure faithfulness, context precision/recall for retrieval pipelines.","whenNotTopPick":"General agent/tool evals — Promptfoo/Braintrust.","categoryIds":["cat-observability"],"depotPickProviderId":"ragas","depotPickRationale":"Capability fit for RAG metrics.","runnersUpIds":["promptfoo","braintrust","galileo"],"weights":{"capabilityFit":0.26,"pricePerformance":0.12,"latencyReliability":0.12,"developerDx":0.22,"privacyPosture":0.12,"lockInRisk":0.08,"maturity":0.08},"methodologyNotes":"Capability fit for RAG metrics.","tags":["rag"],"depth":{"whenNotToUse":{"text":"General agent/tool evals — Promptfoo/Braintrust.","source":"editorial `whenNotTopPick` (data/jtbds.json)"},"whenNotToUseReason":null,"runnerUp":{"providerId":"promptfoo","name":"Promptfoo","overallForJob":8.43,"pickOverallForJob":8.29,"beatsPickOn":["capabilityFit","latencyReliability","developerDx","maturity"],"source":"first resolvable entry of the job's editorially ranked `runnersUpIds`; scores are stored job-weighted scores"},"runnerUpReason":null,"lockIn":{"pickLockInRisk":2,"runnerUpLockInRisk":2.5,"scale":"0–10 stored curated estimate; higher = more lock-in (worse)","pickExitSignals":["open source (provider tag `oss`)"],"runnerUpExitSignals":["open source (provider tag `oss`)"],"pickNotes":[],"selfHostFallback":[{"id":"ragas","name":"Ragas","license":"Apache-2.0","sourceUrl":"https://raw.githubusercontent.com/vibrantlabsai/ragas/main/LICENSE"}],"migration":"The pick has lower stored lock-in risk than runner-up Promptfoo (2 vs 2.5). Pick exit signals: open source (provider tag `oss`). Self-host fallback listed for this job: Ragas.","migrationReason":null,"derivedFrom":"stored lockInRisk scores, provider tags (oss, self-host, openai-compat, open-models), free rows that list this job, and verbatim provider notes. No migration guide was invented."}},"freshness":{"pick":{"score":{"asOf":"2026-09-14","ageDays":43,"stale":true,"staleAfterDays":30,"reason":null},"lastReviewed":{"asOf":"2026-09-14","ageDays":43,"stale":true,"staleAfterDays":30,"reason":null},"sourceCheck":{"checkedAt":"2026-10-27","sourceUrls":3,"sourcesReachable":3,"pricingRows":0,"pricingRowsReconfirmed":0,"note":"A source re-check confirms the page is reachable and, for pricing rows, that the stored amount and SKU name appear on it. It does not re-derive or change the score.","reason":null}}}}