{"id":"public-agent-benchmarks","slug":"public-agent-benchmarks","name":"Public Agent Capability Benchmarks (SWE-bench, τ-bench, GAIA, OSWorld, WebArena, Terminal-Bench, BFCL, MLE-bench)","shortName":"Public agent capability benchmarks","problemStatement":"Select and run standardized, contamination-checked public agent benchmarks matched to your agent's task taxonomy (coding, tool-policy dialogue, web/GUI actions, shell/CLI, function-calling, ML engineering) so capability claims are reproducible outside your private test set.","whenNotTopPick":"Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards); hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases; hosted dataset + scorer UI for team review — Braintrust.","categoryIds":["cat-observability","cat-sandboxes-browsers"],"depotPickProviderId":"promptfoo","depotPickRationale":"Promptfoo runs local, declarative YAML/CLI evaluation matrices across providers and models (Apache-2.0, $0 OSS CLI) with custom Python/JS graders and Docker/sandbox hooks — ideal for wrapping public benchmark subsets (BFCL, GAIA, τ-bench, SWE-bench Verified slices) in repeatable runs without SaaS lock-in.","runnersUpIds":["braintrust","e2b","modal","weights-biases"],"weights":{"capabilityFit":0.26,"pricePerformance":0.18,"latencyReliability":0.14,"developerDx":0.18,"privacyPosture":0.1,"lockInRisk":0.08,"maturity":0.06},"methodologyNotes":"Maps agent task types to official public benchmarks (no invented scores): SWE-bench Verified (https://www.swebench.com/) for repo-level bug fixing; τ-bench (https://github.com/sierra-research/tau-bench) for multi-turn tool + policy compliance; GAIA (https://huggingface.co/spaces/gaia-benchmark/leaderboard) for tool-augmented general assistant reasoning; WebArena / BrowserGym (https://webarena.dev/ · https://github.com/ServiceNow/BrowserGym) for realistic web apps; OSWorld (https://os-world.github.io/) for desktop GUI computer-use; Terminal-Bench (https://tbench.ai/) for CLI/sysadmin tasks; Berkeley Function-Calling Leaderboard BFCL (https://gorilla.cs.berkeley.edu/leaderboard.html) for tool schema AST accuracy; MLE-bench (https://github.com/openai/mle-bench) for autonomous ML engineering.","tags":["evals","benchmarks","swe-bench","tau-bench","gaia","osworld","webarena","terminal-bench","bfcl","mle-bench"],"scoreMargin":0.22,"isNearTie":true,"pickConfidenceBand":"near_tie_co_pick","nearTie":{"isNearTie":true,"scoreMargin":0.22,"threshold":0.35,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"braintrust","coPickProviderName":"Braintrust","coPickScore":8.22,"decidingFactor":"pricePerformance (9.2 vs 7.5)","methodologyWarning":"Top pick (8.44) and #2 runner-up (8.22) are separated by 0.22 points (<= 0.35 weight-sensitivity band). Treat them as co-picks and use the workload tie-breaker rule below rather than the raw composite decimal.","tieBreakerRule":"Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards); hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases; hosted dataset + scorer UI for team review — Braintrust."},"evidenceStatus":{"decisionRole":"independent_prior_and_candidate_generator","positioning":"The independent, agent-readable prior for choosing AI infrastructure — not a universal router, generic marketplace, or synthetic leaderboard.","freshnessThresholdDays":30,"freshness_policy":{"policy_version":"agentdepot.freshness_policy.v1","threshold_days":30},"freshness_state":"fresh","editorial_freshness_state":"fresh","unqualified_current_pick_allowed":true,"pick_qualification":"qualified_current_pick","promotionClass":"deterministic_source_reverified","evidenceClass":"editorial_prior","staleDowngraded":false,"effectiveConfidence":"high","confidenceHalfWidth":0.5,"pickScoreAgeDays":3,"editorialJudgmentAgeDays":9,"pricingAgeDays":3,"isPickScoreStale":false,"isPricingStale":false,"editorialJudgmentAsOf":"2026-10-19","pricingVerifiedAt":"2026-10-25","outcomeVerifiedAt":null,"effectiveVerifiedAsOf":"2026-10-25","downgradeReasons":[],"stalenessWarning":"Editorial judgment and deterministic pricing verification are both within the 30-day freshness threshold.","promotionGate":{"deterministicPricingCheck":true,"approvedScoreReaffirm":true,"pendingHumanReviewBlock":false,"zeroUnverifiedSkus":true,"costEfficiencyConsistent":true,"lockInAndReliabilityConsistent":true,"secondModelOrRuleAudit":true,"countersignReceiptIds":[],"otsAttestationHead":"b32634d5784cbfe663be0a16cce4f60a3ff1c3d4490437be00361cb4ae5d5d59","sourceCheckDate":"2026-10-27","activePricingRowsCount":0,"freeResourcesCount":1,"runnerUpPricingRowsCount":1},"claimProvenance":{"claim_id":"claim:public-agent-benchmarks:promptfoo:depot-pick","provider_id":"promptfoo","jtbd_id":"public-agent-benchmarks","primary_url":"https://www.promptfoo.dev/pricing","source_type":"primary_pricing","source_revision":"sha256:1ae69609cf01d1ca","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.promptfoo.dev/pricing","final_url":"https://www.promptfoo.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:1ae69609cf01d1ca","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.promptfoo.dev","final_url":"https://www.promptfoo.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:0f3e1cf49c5b44e0","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://qaskills.sh/blog/llm-evals-comparison-openai-promptfoo-ragas","final_url":"https://qaskills.sh/blog/llm-evals-comparison-openai-promptfoo-ragas","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_benchmark_or_status","is_claim_specific":true,"source_revision":"sha256:0f488916f20c16b9","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"verifiedOutcomeSummary":{"totalVerifiedReceipts":0,"byEvidenceTier":{"E0_ASSERTION":0,"E1_SOURCE_BOUND":0,"E2_DELIVERY":0,"E3_INDEPENDENT_OUTCOME":0,"E4_REPLICATED_OUTCOME":0},"bySubjectLayer":{"provider":0,"product":0,"model_version":0,"runtime_route":0}}},"pickScoreDetails":{"scoreRange":[7.94,8.94],"formattedScore":"8.4 ± 0.5 (7.9–8.9 · editorial_prior)","evidenceClass":"editorial_prior","promotionClass":"deterministic_source_reverified","staleDowngraded":false,"freshState":"fresh","freshness_state":"fresh","effectiveConfidence":"high","confidenceInterval":{"low":7.94,"mid":8.44,"high":8.94,"halfWidth":0.5,"confidenceLevel":0.8,"basis":"deterministic_source_reverified_interval_0.5"},"editorialJudgmentAsOf":"2026-10-28","sourceAgeDays":9,"pricingVerifiedAt":"2026-10-25","pricingAgeDays":3,"outcomeVerifiedAt":null,"effectiveVerifiedAsOf":"2026-10-25","dissentingEvidence":["Less polished product UI than Braintrust","Enterprise features gated","You bring judge costs","When not to use top pick: Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards); hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases; hosted dataset + scorer UI for team review — Braintrust.","Runner-up Braintrust (braintrust) is the primary alternative when constraints disqualify Promptfoo."],"whatWouldChangePick":["Caller constraint matching: Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards); hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases; hosted dataset + scorer UI for team review — Braintrust.","Live quota or rate-limit exhaustion on promptfoo (pass ?exhausted=promptfoo or liveQuota.exhaustedProviders=[\"promptfoo\"] to promote braintrust).","Active subscription or enterprise commit on braintrust combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.promptfoo.dev or unverified SKU drift detected by npm run check:sources."],"claimProvenance":{"claim_id":"claim:public-agent-benchmarks:promptfoo:depot-pick","provider_id":"promptfoo","jtbd_id":"public-agent-benchmarks","primary_url":"https://www.promptfoo.dev/pricing","source_type":"primary_pricing","source_revision":"sha256:1ae69609cf01d1ca","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.promptfoo.dev/pricing","final_url":"https://www.promptfoo.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:1ae69609cf01d1ca","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.promptfoo.dev","final_url":"https://www.promptfoo.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:0f3e1cf49c5b44e0","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://qaskills.sh/blog/llm-evals-comparison-openai-promptfoo-ragas","final_url":"https://qaskills.sh/blog/llm-evals-comparison-openai-promptfoo-ragas","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_benchmark_or_status","is_claim_specific":true,"source_revision":"sha256:0f488916f20c16b9","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"calibratedScore":{"point":8.44,"band":[7.94,8.94],"interval":{"lower":7.94,"upper":8.94,"half_width":0.5},"score_kind":"source_bound_prior_band","evidence_class":"source_bound","confidence":"medium","confidence_ceiling":"medium","age_days":3,"dissent":["Less polished product UI than Braintrust","Enterprise features gated","You bring judge costs","When not to use top pick: Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards); hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases; hosted dataset + scorer UI for team review — Braintrust.","Runner-up Braintrust (braintrust) is the primary alternative when constraints disqualify Promptfoo."],"what_would_change_this":["Caller constraint matching: Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards); hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases; hosted dataset + scorer UI for team review — Braintrust.","Live quota or rate-limit exhaustion on promptfoo (pass ?exhausted=promptfoo or liveQuota.exhaustedProviders=[\"promptfoo\"] to promote braintrust).","Active subscription or enterprise commit on braintrust combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.promptfoo.dev or unverified SKU drift detected by npm run check:sources."],"subject_layer":"provider"},"subject_ref":{"layer":"provider","provider_id":"promptfoo","product_id":"promptfoo-core","model_id":null,"version":null,"route_class":null,"inheritance_policy":"no_cross_layer_inheritance"}},"runtimeRoutes":[{"routeId":"public-agent-benchmarks:promptfoo:primary","roleForJob":"depot_pick","provider":{"id":"promptfoo","slug":"promptfoo","name":"Promptfoo","website":"https://www.promptfoo.dev","status":"active"},"model":{"representativeModelId":"Promptfoo Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.promptfoo.dev","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://www.promptfoo.dev (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Promptfoo Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":true,"freeTierSummary":"Promptfoo (oss, asOf 2026-10-25)","rateLimitsSummary":"See /api/limits?provider=promptfoo","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global","us"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":7.8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":9,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"braintrust"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:promptfoo:promptfoo-primary:self_hosted_or_oss_compatible:global","provider_id":"promptfoo","product_id":"promptfoo-core","model_id":"promptfoo-primary","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"global","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:promptfoo:promptfoo-primary:direct_api:global","provider_id":"promptfoo","product_id":"promptfoo-core","model_id":"promptfoo-primary","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"public-agent-benchmarks:braintrust:primary","roleForJob":"runner_up","provider":{"id":"braintrust","slug":"braintrust","name":"Braintrust","website":"https://www.braintrust.dev","status":"active"},"model":{"representativeModelId":"Braintrust Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.braintrust.dev","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.braintrust.dev (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Braintrust Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"Starter: Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - ","subscriptionFirstEligible":true,"documentedPlansCount":2},"regions":{"supportedRegions":["us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.6,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"promptfoo"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:managed_cloud_service:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:direct_api:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"direct_api","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"public-agent-benchmarks:e2b:primary","roleForJob":"runner_up","provider":{"id":"e2b","slug":"e2b","name":"E2B","website":"https://e2b.dev","status":"active"},"model":{"representativeModelId":"Pro","skuId":"e2b-pro-usd-per-month","unit":"usd_per_month","amountUsd":150,"sourceUrl":"https://e2b.dev/pricing","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://e2b.dev (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"E2B Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":true,"freeTierSummary":"E2B SDK (oss, asOf 2026-10-25)","rateLimitsSummary":"Hobby: Hobby FREE /$100 of usage in credits + Usage Costs … $100 one-time usage credit Sessions u","subscriptionFirstEligible":true,"documentedPlansCount":4},"regions":{"supportedRegions":["us","eu","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.6,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.5,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":13,"unverifiedSkus":0,"liveFailoverTargetId":"promptfoo"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:e2b:Pro:self_hosted_or_oss_compatible:us","provider_id":"e2b","product_id":"e2b-core","model_id":"Pro","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"us","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:e2b:Pro:direct_api:us","provider_id":"e2b","product_id":"e2b-core","model_id":"Pro","version":"2026-10-28","route_class":"direct_api","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"public-agent-benchmarks:modal:primary","roleForJob":"runner_up","provider":{"id":"modal","slug":"modal","name":"Modal","website":"https://modal.com","status":"active"},"model":{"representativeModelId":"Nvidia B300","skuId":"modal-nvidia-b300-usd-per-gpu-second","unit":"usd_per_gpu_second","amountUsd":0.001972,"sourceUrl":"https://modal.com/pricing","asOf":"2026-10-27"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://modal.com (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Modal Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=modal","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.6,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":7.8,"zeroDataRetentionAvailable":false,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":18,"unverifiedSkus":0,"liveFailoverTargetId":"promptfoo"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:modal:Nvidia B300:managed_cloud_service:us","provider_id":"modal","product_id":"modal-core","model_id":"Nvidia B300","version":"2026-10-27","route_class":"managed_cloud_service","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:modal:Nvidia B300:direct_api:us","provider_id":"modal","product_id":"modal-core","model_id":"Nvidia B300","version":"2026-10-27","route_class":"direct_api","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]}],"depth":{"whenNotToUse":{"text":"Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards); hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases; hosted dataset + scorer UI for team review — Braintrust.","source":"editorial `whenNotTopPick` (data/jtbds.json)"},"whenNotToUseReason":null,"runnerUp":{"providerId":"braintrust","name":"Braintrust","overallForJob":8.22,"pickOverallForJob":8.44,"beatsPickOn":["capabilityFit","latencyReliability","maturity"],"source":"first resolvable entry of the job's editorially ranked `runnersUpIds`; scores are stored job-weighted scores"},"runnerUpReason":null,"lockIn":{"pickLockInRisk":2.5,"runnerUpLockInRisk":5,"scale":"0–10 stored curated estimate; higher = more lock-in (worse)","pickExitSignals":["open source (provider tag `oss`)"],"runnerUpExitSignals":[],"pickNotes":[],"selfHostFallback":[],"migration":"The pick has lower stored lock-in risk than runner-up Braintrust (2.5 vs 5). Pick exit signals: open source (provider tag `oss`).","migrationReason":null,"derivedFrom":"stored lockInRisk scores, provider tags (oss, self-host, openai-compat, open-models), free rows that list this job, and verbatim provider notes. No migration guide was invented."}},"freshness":{"pick":{"score":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"lastReviewed":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"sourceCheck":{"checkedAt":"2026-10-28","sourceUrls":3,"sourcesReachable":3,"pricingRows":0,"pricingRowsReconfirmed":0,"note":"A source re-check confirms the page is reachable and, for pricing rows, that the stored amount and SKU name appear on it. It does not re-derive or change the score.","reason":null}}},"whenNotTopPickRules":[{"ifConstraint":"escape_hatch_clause_1","recommendedProviderId":"e2b","activeModelOrSku":"e2b-sandbox-1vcpu-per-s","reason":"Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards)"},{"ifConstraint":"exit:self_host_or_open_weights","recommendedProviderId":"weights-biases","activeModelOrSku":"wandb-teams","reason":"hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases"},{"ifConstraint":"escape_hatch_clause_3","recommendedProviderId":"braintrust","activeModelOrSku":"braintrust-pro","reason":"hosted dataset + scorer UI for team review — Braintrust."}],"bootstrap":{"sdkInstall":{"npm":"npm install promptfoo","pip":null},"installCommand":"npx promptfoo@latest init","envVars":["OPENAI_API_KEY"],"activeModelOrSku":"promptfoo-oss","canonicalSnippet":"# Run red-team & eval suite via CLI\nnpx promptfoo@latest eval -c promptfooconfig.yaml","docsUrl":"https://www.promptfoo.dev/docs/intro/","bootstrapReason":"Node CLI & library (promptfoo on npm); no separate Python package required.","authType":"api_key","signupUrl":"https://www.promptfoo.dev","apiKeyUrl":"https://www.promptfoo.dev","mcpServer":null,"cliInit":"npx promptfoo@latest init","verifyCommand":"test -n \"$OPENAI_API_KEY\" && echo \"OPENAI_API_KEY configured\""},"quickstartSpec":{"installCommand":"npx promptfoo@latest init","envVars":["OPENAI_API_KEY"],"activeModelOrSku":"promptfoo-oss","canonicalSnippet":"# Run red-team & eval suite via CLI\nnpx promptfoo@latest eval -c promptfooconfig.yaml","sdkInstall":{"npm":"npm install promptfoo","pip":null},"verifyCommand":"test -n \"$OPENAI_API_KEY\" && echo \"OPENAI_API_KEY configured\"","apiKeyUrl":"https://www.promptfoo.dev","docsUrl":"https://www.promptfoo.dev/docs/intro/","mcpServer":null,"bootstrapReason":"Node CLI & library (promptfoo on npm); no separate Python package required."},"operationalGotchas":[{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Less polished product UI than Braintrust","sourceDate":"2026-10-28"},{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Enterprise features gated","sourceDate":"2026-10-28"}],"relatedPacks":[]}