{"id":"automated-red-teaming-self-hardening","slug":"automated-red-teaming-self-hardening","name":"Continuous Adversarial Red-Teaming, Prompt-Injection & Policy Self-Hardening","shortName":"Automated red-teaming & self-hardening","problemStatement":"Continuously generate adversarial attacks (indirect prompt injection in untrusted tool outputs, PII exfiltration, unauthorized tool calls, jailbreaks) against your agent, patch system guardrails, and verify that attack success rate drops without spiking false-positive refusals.","whenNotTopPick":"Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI.","categoryIds":["cat-observability","cat-observability"],"depotPickProviderId":"promptfoo","depotPickRationale":"Promptfoo includes an open-source automated red-team generator (`promptfoo redteam`, Apache-2.0 CLI) covering indirect prompt injection, tool/BFLA abuse, SSRF, and PII leaks, letting agents run closed-loop adversarial probes in CI before every deploy.","runnersUpIds":["lakera","guardrails-ai","patronus"],"weights":{"capabilityFit":0.28,"pricePerformance":0.16,"latencyReliability":0.14,"developerDx":0.16,"privacyPosture":0.14,"lockInRisk":0.06,"maturity":0.06},"methodologyNotes":"Track both Attack Success Rate (`ASR`, lower is better) on adversarial probes and False Positive Refusal Rate (`FPR`, lower is better) on benign golden tasks so hardening never lobotomizes the agent.","tags":["learning","red-teaming","security","prompt-injection","guardrails","self-hardening","self-improvement"],"scoreMargin":0.15,"isNearTie":true,"pickConfidenceBand":"near_tie_co_pick","nearTie":{"isNearTie":true,"scoreMargin":0.15,"threshold":0.35,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"lakera","coPickProviderName":"Lakera Guard","coPickScore":8.31,"decidingFactor":"pricePerformance (9.2 vs 7.8)","methodologyWarning":"Top pick (8.46) and #2 runner-up (8.31) are separated by 0.15 points (<= 0.35 weight-sensitivity band). Treat them as co-picks and use the workload tie-breaker rule below rather than the raw composite decimal.","tieBreakerRule":"Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI."},"evidenceStatus":{"decisionRole":"independent_prior_and_candidate_generator","positioning":"The independent, agent-readable prior for choosing AI infrastructure — not a universal router, generic marketplace, or synthetic leaderboard.","freshnessThresholdDays":30,"freshness_policy":{"policy_version":"agentdepot.freshness_policy.v1","threshold_days":30},"freshness_state":"fresh","editorial_freshness_state":"fresh","unqualified_current_pick_allowed":true,"pick_qualification":"qualified_current_pick","promotionClass":"deterministic_source_reverified","evidenceClass":"editorial_prior","staleDowngraded":false,"effectiveConfidence":"medium_reverified","confidenceHalfWidth":0.5,"pickScoreAgeDays":3,"editorialJudgmentAgeDays":9,"pricingAgeDays":3,"isPickScoreStale":false,"isPricingStale":false,"editorialJudgmentAsOf":"2026-10-19","pricingVerifiedAt":"2026-10-25","outcomeVerifiedAt":null,"effectiveVerifiedAsOf":"2026-10-25","downgradeReasons":[],"stalenessWarning":"Editorial judgment and deterministic pricing verification are both within the 30-day freshness threshold.","promotionGate":{"deterministicPricingCheck":true,"approvedScoreReaffirm":true,"pendingHumanReviewBlock":false,"zeroUnverifiedSkus":true,"costEfficiencyConsistent":true,"lockInAndReliabilityConsistent":true,"secondModelOrRuleAudit":true,"countersignReceiptIds":[],"otsAttestationHead":"b32634d5784cbfe663be0a16cce4f60a3ff1c3d4490437be00361cb4ae5d5d59","sourceCheckDate":"2026-10-27","activePricingRowsCount":0,"freeResourcesCount":1,"runnerUpPricingRowsCount":1},"claimProvenance":{"claim_id":"claim:automated-red-teaming-self-hardening:promptfoo:depot-pick","provider_id":"promptfoo","jtbd_id":"automated-red-teaming-self-hardening","primary_url":"https://www.promptfoo.dev/pricing","source_type":"primary_pricing","source_revision":"sha256:1ae69609cf01d1ca","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.promptfoo.dev/pricing","final_url":"https://www.promptfoo.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:1ae69609cf01d1ca","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.promptfoo.dev","final_url":"https://www.promptfoo.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:0f3e1cf49c5b44e0","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://qaskills.sh/blog/llm-evals-comparison-openai-promptfoo-ragas","final_url":"https://qaskills.sh/blog/llm-evals-comparison-openai-promptfoo-ragas","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_benchmark_or_status","is_claim_specific":true,"source_revision":"sha256:0f488916f20c16b9","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"verifiedOutcomeSummary":{"totalVerifiedReceipts":0,"byEvidenceTier":{"E0_ASSERTION":0,"E1_SOURCE_BOUND":0,"E2_DELIVERY":0,"E3_INDEPENDENT_OUTCOME":0,"E4_REPLICATED_OUTCOME":0},"bySubjectLayer":{"provider":0,"product":0,"model_version":0,"runtime_route":0}}},"pickScoreDetails":{"scoreRange":[7.96,8.96],"formattedScore":"8.5 ± 0.5 (8.0–9.0 · editorial_prior)","evidenceClass":"editorial_prior","promotionClass":"deterministic_source_reverified","staleDowngraded":false,"freshState":"fresh","freshness_state":"fresh","effectiveConfidence":"medium_reverified","confidenceInterval":{"low":7.96,"mid":8.46,"high":8.96,"halfWidth":0.5,"confidenceLevel":0.8,"basis":"deterministic_source_reverified_interval_0.5"},"editorialJudgmentAsOf":"2026-10-28","sourceAgeDays":9,"pricingVerifiedAt":"2026-10-25","pricingAgeDays":3,"outcomeVerifiedAt":null,"effectiveVerifiedAsOf":"2026-10-25","dissentingEvidence":["Less polished product UI than Braintrust","Enterprise features gated","You bring judge costs","When not to use top pick: Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI.","Runner-up Lakera Guard (lakera) is the primary alternative when constraints disqualify Promptfoo."],"whatWouldChangePick":["Caller constraint matching: Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI.","Live quota or rate-limit exhaustion on promptfoo (pass ?exhausted=promptfoo or liveQuota.exhaustedProviders=[\"promptfoo\"] to promote lakera).","Active subscription or enterprise commit on lakera combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.promptfoo.dev or unverified SKU drift detected by npm run check:sources."],"claimProvenance":{"claim_id":"claim:automated-red-teaming-self-hardening:promptfoo:depot-pick","provider_id":"promptfoo","jtbd_id":"automated-red-teaming-self-hardening","primary_url":"https://www.promptfoo.dev/pricing","source_type":"primary_pricing","source_revision":"sha256:1ae69609cf01d1ca","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.promptfoo.dev/pricing","final_url":"https://www.promptfoo.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:1ae69609cf01d1ca","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.promptfoo.dev","final_url":"https://www.promptfoo.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:0f3e1cf49c5b44e0","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://qaskills.sh/blog/llm-evals-comparison-openai-promptfoo-ragas","final_url":"https://qaskills.sh/blog/llm-evals-comparison-openai-promptfoo-ragas","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_benchmark_or_status","is_claim_specific":true,"source_revision":"sha256:0f488916f20c16b9","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"calibratedScore":{"point":8.46,"band":[7.96,8.96],"interval":{"lower":7.96,"upper":8.96,"half_width":0.5},"score_kind":"source_bound_prior_band","evidence_class":"source_bound","confidence":"low","confidence_ceiling":"medium","age_days":3,"dissent":["Less polished product UI than Braintrust","Enterprise features gated","You bring judge costs","When not to use top pick: Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI.","Runner-up Lakera Guard (lakera) is the primary alternative when constraints disqualify Promptfoo."],"what_would_change_this":["Caller constraint matching: Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI.","Live quota or rate-limit exhaustion on promptfoo (pass ?exhausted=promptfoo or liveQuota.exhaustedProviders=[\"promptfoo\"] to promote lakera).","Active subscription or enterprise commit on lakera combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.promptfoo.dev or unverified SKU drift detected by npm run check:sources."],"subject_layer":"provider"},"subject_ref":{"layer":"provider","provider_id":"promptfoo","product_id":"promptfoo-core","model_id":null,"version":null,"route_class":null,"inheritance_policy":"no_cross_layer_inheritance"}},"runtimeRoutes":[{"routeId":"automated-red-teaming-self-hardening:promptfoo:primary","roleForJob":"depot_pick","provider":{"id":"promptfoo","slug":"promptfoo","name":"Promptfoo","website":"https://www.promptfoo.dev","status":"active"},"model":{"representativeModelId":"Promptfoo Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.promptfoo.dev","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://www.promptfoo.dev (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Promptfoo Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":true,"freeTierSummary":"Promptfoo (oss, asOf 2026-10-25)","rateLimitsSummary":"See /api/limits?provider=promptfoo","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global","us"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":7.8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":9,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"lakera"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:promptfoo:promptfoo-primary:self_hosted_or_oss_compatible:global","provider_id":"promptfoo","product_id":"promptfoo-core","model_id":"promptfoo-primary","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"global","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:promptfoo:promptfoo-primary:direct_api:global","provider_id":"promptfoo","product_id":"promptfoo-core","model_id":"promptfoo-primary","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"automated-red-teaming-self-hardening:lakera:primary","roleForJob":"runner_up","provider":{"id":"lakera","slug":"lakera","name":"Lakera Guard","website":"https://www.lakera.ai","status":"active"},"model":{"representativeModelId":"Lakera Guard Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.lakera.ai","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"direct_first_party_api","endpointPattern":"https://www.lakera.ai (direct first party api)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Lakera Guard Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=lakera","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["eu","us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":true,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.4,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"promptfoo"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:lakera:lakera-primary:direct_first_party_api:eu","provider_id":"lakera","product_id":"lakera-core","model_id":"lakera-primary","version":"2026-10-28","route_class":"direct_first_party_api","region":"eu","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:lakera:lakera-primary:direct_api:eu","provider_id":"lakera","product_id":"lakera-core","model_id":"lakera-primary","version":"2026-10-28","route_class":"direct_api","region":"eu","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"automated-red-teaming-self-hardening:guardrails-ai:primary","roleForJob":"runner_up","provider":{"id":"guardrails-ai","slug":"guardrails-ai","name":"Guardrails AI","website":"https://guardrailsai.com","status":"active"},"model":{"representativeModelId":"Guardrails AI Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://guardrailsai.com","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://guardrailsai.com (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Guardrails AI Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=guardrails-ai","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global","us"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":true,"mcpServerAvailable":false,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":7.8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.5,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"promptfoo"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:guardrails-ai:guardrails-ai-primary:self_hosted_or_oss_compatible:global","provider_id":"guardrails-ai","product_id":"guardrails-ai-core","model_id":"guardrails-ai-primary","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"global","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:guardrails-ai:guardrails-ai-primary:direct_api:global","provider_id":"guardrails-ai","product_id":"guardrails-ai-core","model_id":"guardrails-ai-primary","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"automated-red-teaming-self-hardening:patronus:primary","roleForJob":"runner_up","provider":{"id":"patronus","slug":"patronus","name":"Patronus AI","website":"https://www.patronus.ai","status":"active"},"model":{"representativeModelId":"Patronus AI Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.patronus.ai","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.patronus.ai (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Patronus AI Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=patronus","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global","us"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":false,"parallelToolCalls":false,"mcpServerAvailable":false,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.5,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"promptfoo"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:patronus:patronus-primary:managed_cloud_service:global","provider_id":"patronus","product_id":"patronus-core","model_id":"patronus-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:patronus:patronus-primary:direct_api:global","provider_id":"patronus","product_id":"patronus-core","model_id":"patronus-primary","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]}],"depth":{"whenNotToUse":{"text":"Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI.","source":"editorial `whenNotTopPick` (data/jtbds.json)"},"whenNotToUseReason":null,"runnerUp":{"providerId":"lakera","name":"Lakera Guard","overallForJob":8.31,"pickOverallForJob":8.46,"beatsPickOn":["capabilityFit","latencyReliability","developerDx"],"source":"first resolvable entry of the job's editorially ranked `runnersUpIds`; scores are stored job-weighted scores"},"runnerUpReason":null,"lockIn":{"pickLockInRisk":2.5,"runnerUpLockInRisk":5.5,"scale":"0–10 stored curated estimate; higher = more lock-in (worse)","pickExitSignals":["open source (provider tag `oss`)"],"runnerUpExitSignals":[],"pickNotes":[],"selfHostFallback":[],"migration":"The pick has lower stored lock-in risk than runner-up Lakera Guard (2.5 vs 5.5). Pick exit signals: open source (provider tag `oss`).","migrationReason":null,"derivedFrom":"stored lockInRisk scores, provider tags (oss, self-host, openai-compat, open-models), free rows that list this job, and verbatim provider notes. No migration guide was invented."}},"freshness":{"pick":{"score":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"lastReviewed":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"sourceCheck":{"checkedAt":"2026-10-28","sourceUrls":3,"sourcesReachable":3,"pricingRows":0,"pricingRowsReconfirmed":0,"note":"A source re-check confirms the page is reachable and, for pricing rows, that the stored amount and SKU name appear on it. It does not re-derive or change the score.","reason":null}}},"whenNotTopPickRules":[{"ifConstraint":"latency:ultra_low_ttft","recommendedProviderId":"lakera","activeModelOrSku":"lakera-guard","reason":"Low-latency inline runtime prompt-injection firewall in production — Lakera Guard"},{"ifConstraint":"escape_hatch_clause_2","recommendedProviderId":"guardrails-ai","activeModelOrSku":"guardrails-ai-standard","reason":"programmable input/output validator rails in Python/TS — Guardrails AI"},{"ifConstraint":"escape_hatch_clause_3","recommendedProviderId":"patronus","activeModelOrSku":"patronus-standard","reason":"managed enterprise hallucination and policy evaluation suites — Patronus AI."}],"bootstrap":{"sdkInstall":{"npm":"npm install promptfoo","pip":null},"installCommand":"npx promptfoo@latest init","envVars":["OPENAI_API_KEY"],"activeModelOrSku":"promptfoo-oss","canonicalSnippet":"# Run red-team & eval suite via CLI\nnpx promptfoo@latest eval -c promptfooconfig.yaml","docsUrl":"https://www.promptfoo.dev/docs/intro/","bootstrapReason":"Node CLI & library (promptfoo on npm); no separate Python package required.","authType":"api_key","signupUrl":"https://www.promptfoo.dev","apiKeyUrl":"https://www.promptfoo.dev","mcpServer":null,"cliInit":"npx promptfoo@latest init","verifyCommand":"test -n \"$OPENAI_API_KEY\" && echo \"OPENAI_API_KEY configured\""},"quickstartSpec":{"installCommand":"npx promptfoo@latest init","envVars":["OPENAI_API_KEY"],"activeModelOrSku":"promptfoo-oss","canonicalSnippet":"# Run red-team & eval suite via CLI\nnpx promptfoo@latest eval -c promptfooconfig.yaml","sdkInstall":{"npm":"npm install promptfoo","pip":null},"verifyCommand":"test -n \"$OPENAI_API_KEY\" && echo \"OPENAI_API_KEY configured\"","apiKeyUrl":"https://www.promptfoo.dev","docsUrl":"https://www.promptfoo.dev/docs/intro/","mcpServer":null,"bootstrapReason":"Node CLI & library (promptfoo on npm); no separate Python package required."},"operationalGotchas":[{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Less polished product UI than Braintrust","sourceDate":"2026-10-28"},{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Enterprise features gated","sourceDate":"2026-10-28"}],"relatedPacks":[],"depotPick":{"id":"promptfoo","slug":"promptfoo","name":"Promptfoo","tagline":"Default OSS eval harness. Braintrust when you want hosted collaboration; Promptfoo when you want local/CI…","overall":8.46},"runnersUp":[{"id":"lakera","slug":"lakera","name":"Lakera Guard","tagline":"Depot pick for a dedicated prompt-injection firewall in front of any LLM. NeMo when you need programmable…","overall":8.31},{"id":"guardrails-ai","slug":"guardrails-ai","name":"Guardrails AI","tagline":"OSS programmable rails peer to NeMo Guardrails. Lakera when you want a hosted injection firewall API…","overall":8.24},{"id":"patronus","slug":"patronus","name":"Patronus AI","tagline":"Layer on when regulated hallucination/safety gates matter more than general CI evals. Lakera for injection…","overall":7.41}]}