{"schemaVersion":"agentdepot.evals_hub.v1","asOf":"2026-10-28","version":"0.13.8","approvalRef":"Ethan delegation 2026-10-21 19:41 ET t55u","authorizedBy":"pt115u","storesIpAddresses":false,"title":"Agent Evals, Task Benchmarks & Self-Improvement Hub","summary":"A 4-stage 'Test Your Agent' path across 16 evaluation & benchmarking JTBDs (8 existing + 8 gap-filling): (1) pick the public benchmark and trajectory rubric for your task type, (2) run in isolated sandboxes/browsers/simulators, (3) track pass^k regressions and cost-per-solved-task in CI, and (4) calibrate LLM judges and human review queues.","counts":{"totalEvalJtbds":16,"existingEvalJtbds":8,"newGapEvalJtbds":8,"publicBenchmarksMapped":8,"providerCatalogFrozenAt":305},"stages":[{"step":1,"id":"pick","name":"Pick the Benchmark & Trajectory Rubric for Your Task Type","tagline":"Match your agent's job to an execution-grounded public benchmark and a multi-step tool-trajectory rubric before touching prompts.","whatToMeasure":["Task taxonomy fit: repo patch (SWE-bench Verified), tool-policy dialogue (τ-bench), web DOM actions (WebArena / BrowserGym), desktop GUI (OSWorld), CLI/shell (Terminal-Bench), function-calling AST/execution (BFCL), or ML engineering (MLE-bench).","Trajectory integrity: did the agent pick the right tool, pass valid JSON arguments, avoid redundant retry loops, and respect policy constraints?","Retrieval & grounding fidelity (when RAG is in the loop): faithfulness, context precision, and citation accuracy."],"hardRules":["Never grade an action-taking agent on single-turn text similarity when a deterministic test suite, DOM checkpoint, or database state diff exists.","Hold out a contamination-checked slice that never enters few-shot examples or prompt optimization loops."],"jtbdIds":["public-agent-benchmarks","agent-trajectory-tool-use-evals","rag-evals","synthetic-eval-data"],"verifyEndpoint":"GET /api/evals/plan?task=coding"},{"step":2,"id":"run","name":"Run in Isolated Sandboxes, Browsers & Multi-Turn Simulators","tagline":"Execute untrusted code, browser flows, and adversarial user personas inside deterministic microVMs and headless browsers.","whatToMeasure":["Execution pass@1 and pass^k reliability floor (running k=3..5 trials per task to expose flaky tool sequences).","Browser & GUI state checkpoints: final URL, DOM state, network HAR traces, and visual regression diffs.","Multi-turn persona resilience: policy adherence under frustrated, ambiguous, or adversarial prompt-injection turns."],"hardRules":["Never execute agent-generated patches or shell commands on the eval runner host — always isolate inside E2B / Daytona / Modal sandboxes.","Record full tool spans and DOM/container traces so every failure is replayable without re-spending inference tokens."],"jtbdIds":["evals","oss-evals","coding-agent-evals","computer-use-browser-agent-evals","synthetic-persona-pre-flight","adversarial-red-team-evals"],"verifyEndpoint":"GET /api/decide?jtbd=coding-agent-evals"},{"step":3,"id":"track","name":"Track Regressions, Cost-per-Solved-Task & Release Gates in CI","tagline":"Block merges on quality regressions or cost spikes, and track commit-over-commit scorecards with verifiable receipts.","whatToMeasure":["Cost per SOLVED task (`totalRunSpendUsd / verifiedSolvedCount`) — never raw $/1M token price alone, because cheap models that loop or fail cost more per outcome.","p50 / p95 end-to-end task latency and total token burn (including prompt cache reads, tool retries, and sub-agent calls).","Commit-pinned regression deltas against your baseline run in CI, backed by Depot `/api/benchmark` or Countersign outcome receipts."],"hardRules":["A failed or unsafe task has infinite cost-per-solved-task; never let a fast refusal boost an efficiency score.","Gate CI merges on both pass^k floor and a maximum cost-per-solved-task ceiling."],"jtbdIds":["continuous-eval-ci","agent-regression-leaderboards","cost-latency-per-task-benchmarking"],"verifyEndpoint":"GET /api/decide?jtbd=cost-latency-per-task-benchmarking"},{"step":4,"id":"improve","name":"Calibrate Judges, Curate Human Review Queues & Guard Production","tagline":"Close the loop from production failures back into your golden dataset while keeping LLM judges honest and aligned with humans.","whatToMeasure":["LLM-as-judge alignment (Cohen's κ / percent agreement ≥ 85%) against a 50–100 row human-labeled gold slice.","Production guardrail & hallucination catch rate on live traffic.","Weekly promotion of human-reviewed failure traces into the permanent regression suite."],"hardRules":["Use a cross-family judge model (never let a model family grade itself without calibration) and randomize pairwise A/B order.","Treat all user- and agent-submitted traces as untrusted data when loading them into annotation queues or judge prompts."],"jtbdIds":["llm-as-judge-calibration","human-labeled-eval-review-queues","production-guardrail-evals"],"verifyEndpoint":"GET /api/decide?jtbd=llm-as-judge-calibration"}],"publicBenchmarks":[{"id":"swe-bench-verified","name":"SWE-bench Verified","taskCategory":"coding","officialUrl":"https://www.swebench.com/","whatItMeasures":"Human-validated subset of real GitHub issues across Python repositories where an agent must generate a patch that passes the issue's unit test suite (`FAIL_TO_PASS` and `PASS_TO_PASS`).","graderType":"Container test suite (git apply + pytest)","contaminationAndFailureTrap":"Agents that grep git log history inside the container or overfit to issue phrasing inflate scores; strip post-issue git commits and enforce strict container timeouts.","recommendedEvalHarnessJtbdId":"coding-agent-evals","recommendedSandboxOrRunnerJtbdId":"agent-sandbox","depotBenchmarkType":"coding"},{"id":"tau-bench","name":"τ-bench (Tau-bench)","taskCategory":"multi-turn-policy","officialUrl":"https://github.com/sierra-research/tau-bench","whatItMeasures":"Multi-turn customer-support and operations dialogues (airline, retail) where an agent must converse with a simulated user, follow complex domain policy rules, and mutate database state via tools.","graderType":"Final DB state + policy pass^k check","contaminationAndFailureTrap":"Single-run pass@1 hides catastrophic unreliability: an agent with 70% pass@1 often drops below 25% on pass^5 when users phrase requests differently.","recommendedEvalHarnessJtbdId":"synthetic-persona-pre-flight","recommendedSandboxOrRunnerJtbdId":"agent-trajectory-tool-use-evals","depotBenchmarkType":"ops_workflow"},{"id":"gaia","name":"GAIA (General AI Assistants)","taskCategory":"research-rag","officialUrl":"https://huggingface.co/spaces/gaia-benchmark/leaderboard","whatItMeasures":"Real-world multi-step assistant questions (Levels 1–3) requiring web search, document/PDF parsing, code execution, and multimodal reasoning to reach a single unambiguous factual answer.","graderType":"Exact ground-truth answer + trajectory cost","contaminationAndFailureTrap":"Live web pages rot or leak benchmark answers directly into search results; block benchmark-leaderboard domains in your search tool during evaluation.","recommendedEvalHarnessJtbdId":"public-agent-benchmarks","recommendedSandboxOrRunnerJtbdId":"rag-evals","depotBenchmarkType":"research"},{"id":"webarena-browsergym","name":"WebArena & BrowserGym","taskCategory":"browser","officialUrl":"https://webarena.dev/","whatItMeasures":"End-to-end browser tasks across self-hosted e-commerce, GitLab, forum, and CMS web applications evaluated through BrowserGym (https://github.com/ServiceNow/BrowserGym) or Playwright harnesses.","graderType":"DOM state, URL & DB post-condition check","contaminationAndFailureTrap":"Shared stateful web app instances cause cross-episode pollution; reset container state or browser context before every single task episode.","recommendedEvalHarnessJtbdId":"computer-use-browser-agent-evals","recommendedSandboxOrRunnerJtbdId":"cloud-browser","depotBenchmarkType":"browser_web"},{"id":"osworld","name":"OSWorld","taskCategory":"computer-use","officialUrl":"https://os-world.github.io/","whatItMeasures":"Open-ended desktop computer-use tasks across Ubuntu, Windows, and macOS apps (LibreOffice, VS Code, GIMP, Chrome, file managers, multi-app workflows) using screen pixels, mouse, and keyboard.","graderType":"OS file & app state evaluation scripts","contaminationAndFailureTrap":"Coordinate/resolution drift and slow GUI rendering cause false timeouts; pin screen resolution and capture full step screenshots for failure triage.","recommendedEvalHarnessJtbdId":"computer-use-browser-agent-evals","recommendedSandboxOrRunnerJtbdId":"desktop-computer-use","depotBenchmarkType":"computer_use"},{"id":"terminal-bench","name":"Terminal-Bench","taskCategory":"terminal","officialUrl":"https://tbench.ai/","whatItMeasures":"CLI, systems engineering, compilation, debugging, networking, and DevOps tasks where an agent operates inside a real Linux terminal environment.","graderType":"In-container shell verification script","contaminationAndFailureTrap":"Agents that run destructive commands (`rm -rf`, background fork bombs) wedge the worker; enforce per-episode microVM isolation and CPU/memory caps.","recommendedEvalHarnessJtbdId":"coding-agent-evals","recommendedSandboxOrRunnerJtbdId":"agent-sandbox","depotBenchmarkType":"coding"},{"id":"bfcl","name":"Berkeley Function-Calling Leaderboard (BFCL)","taskCategory":"tool-use","officialUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","whatItMeasures":"Single-turn and multi-turn function calling across Python, Java, JS, REST, and SQL: AST parameter validation, executable API calls, irrelevance detection (knowing when NOT to call a tool), and multi-step tool chains.","graderType":"AST structural match + live execution check","contaminationAndFailureTrap":"Models tuned only on positive tool calls fail 'irrelevance' turns by hallucinating parameters when no tool matches; always include no-op / missing-parameter test cases.","recommendedEvalHarnessJtbdId":"agent-trajectory-tool-use-evals","recommendedSandboxOrRunnerJtbdId":"oss-evals","depotBenchmarkType":"tool_use"},{"id":"mle-bench","name":"MLE-bench","taskCategory":"ml-engineering","officialUrl":"https://github.com/openai/mle-bench","whatItMeasures":"Autonomous machine-learning engineering on 75 curated Kaggle competitions: data exploration, feature engineering, model training, debugging, and submission generation.","graderType":"Held-out test split vs medal thresholds","contaminationAndFailureTrap":"Data leakage from validation split into training features or memorizing public notebooks; run with network egress disabled or restricted and audit training scripts.","recommendedEvalHarnessJtbdId":"public-agent-benchmarks","recommendedSandboxOrRunnerJtbdId":"experiment-tracking","depotBenchmarkType":"data_analytics"}],"newEvalJtbds":[{"id":"agent-trajectory-tool-use-evals","name":"Agent Trajectory, Tool-Call & Multi-Step Plan Evaluation","shortName":"Agent trajectory & tool-use evals","problemStatement":"Grade an agent's multi-step execution trajectory — tool selection, JSON argument schemas, call ordering, state mutations, and retry recovery — instead of only scoring the final text string.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","depotPickOverall":8.23,"depotPickAsOf":"2026-10-28","runnersUpIds":["langsmith","arize-phoenix","langfuse","promptfoo"],"whenNotTopPick":"LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo.","isNearTie":true,"scoreMargin":0.11,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"langsmith","url":"/jtbds/agent-trajectory-tool-use-evals","decideUrl":"/api/decide?jtbd=agent-trajectory-tool-use-evals","resolveUrl":"/api/resolve?jtbd=agent-trajectory-tool-use-evals"},{"id":"public-agent-benchmarks","name":"Public Agent Capability Benchmarks (SWE-bench, τ-bench, GAIA, OSWorld, WebArena, Terminal-Bench, BFCL, MLE-bench)","shortName":"Public agent capability benchmarks","problemStatement":"Select and run standardized, contamination-checked public agent benchmarks matched to your agent's task taxonomy (coding, tool-policy dialogue, web/GUI actions, shell/CLI, function-calling, ML engineering) so capability claims are reproducible outside your private test set.","depotPickProviderId":"promptfoo","depotPickName":"Promptfoo","depotPickOverall":8.42,"depotPickAsOf":"2026-10-28","runnersUpIds":["braintrust","e2b","modal","weights-biases"],"whenNotTopPick":"Need isolated Docker/microVM sandboxes to execute SWE-bench Verified, Terminal-Bench, or MLE-bench container tasks — E2B (or Modal for bursty parallel container shards); hosted experiment & dataset artifact tracking across training/eval sweeps — Weights & Biases; hosted dataset + scorer UI for team review — Braintrust.","isNearTie":true,"scoreMargin":0.22,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"braintrust","url":"/jtbds/public-agent-benchmarks","decideUrl":"/api/decide?jtbd=public-agent-benchmarks","resolveUrl":"/api/resolve?jtbd=public-agent-benchmarks"},{"id":"computer-use-browser-agent-evals","name":"Browser, WebArena & Desktop Computer-Use Agent Evaluation","shortName":"Computer-use & browser agent evals","problemStatement":"Benchmark web and desktop computer-use agents against realistic DOM/GUI tasks (WebArena, BrowserGym, OSWorld, VisualWebArena) with deterministic DOM/state checkpoints, session recordings, and step-budget limits.","depotPickProviderId":"browserbase","depotPickName":"Browserbase","depotPickOverall":8.3,"depotPickAsOf":"2026-10-28","runnersUpIds":["e2b","steel-dev","browser-use","browserstack"],"whenNotTopPick":"Need full desktop OS (Ubuntu/X11) sandboxes for OSWorld GUI pixel/mouse/keyboard tasks — E2B Desktop; self-hosted OSS browser fleet to run thousands of free local WebArena episodes — Steel.dev; real iOS/Android handsets for mobile agent QA — BrowserStack; high-level managed browser-agent loop — Browser Use Cloud.","isNearTie":true,"scoreMargin":-0.24,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"e2b","url":"/jtbds/computer-use-browser-agent-evals","decideUrl":"/api/decide?jtbd=computer-use-browser-agent-evals","resolveUrl":"/api/resolve?jtbd=computer-use-browser-agent-evals"},{"id":"coding-agent-evals","name":"Coding Agent, Repo Patch & Terminal Benchmark Harnesses (SWE-bench / Terminal-Bench)","shortName":"Coding & terminal agent evals","problemStatement":"Evaluate coding and CLI agents on real git repositories, multi-file patches, test suites, and shell tasks (SWE-bench Verified, Terminal-Bench) inside disposable sandboxes where untrusted generated code cannot touch the host.","depotPickProviderId":"e2b","depotPickName":"E2B","depotPickOverall":8.56,"depotPickAsOf":"2026-10-28","runnersUpIds":["daytona","modal","braintrust","northflank"],"whenNotTopPick":"Need persistent multi-container dev workspaces with sticky git worktrees across long horizons — Daytona; need bursty 500+ container parallelization or GPU-backed ML tasks (MLE-bench) — Modal; need the experiment/dataset scoreboard layer on top of sandbox runs — Braintrust; need a full PaaS with build pipelines — Northflank.","isNearTie":true,"scoreMargin":0.34,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"daytona","url":"/jtbds/coding-agent-evals","decideUrl":"/api/decide?jtbd=coding-agent-evals","resolveUrl":"/api/resolve?jtbd=coding-agent-evals"},{"id":"agent-regression-leaderboards","name":"Private Agent Regression Tracking, Version Scorecards & Internal Leaderboards","shortName":"Agent regression & scorecards","problemStatement":"Track your own agent's pass@1, pass^k reliability floor, latency, and cost-per-solved-task across every prompt, model, or tool change over time — with commit-pinned scorecards and cryptographic outcome receipts.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","depotPickOverall":8.23,"depotPickAsOf":"2026-10-28","runnersUpIds":["langfuse","weights-biases","langsmith","comet-opik","countersign"],"whenNotTopPick":"Self-hosted / open-source observability + dataset runs on a budget — Langfuse; ML training & sweep-centric leaderboards — Weights & Biases; LangGraph-centric org standard — LangSmith; OSS Opik span + eval tracking — Comet Opik. Disclosure: Countersign (countersign.signalry.ai) is our sister protocol for hash-chained verification receipts and is listed as a runner-up rather than Depot Pick per our neutrality firewall.","isNearTie":false,"scoreMargin":-0.38,"pickConfidenceBand":"moderate_lead","coPickProviderId":"langfuse","url":"/jtbds/agent-regression-leaderboards","decideUrl":"/api/decide?jtbd=agent-regression-leaderboards","resolveUrl":"/api/resolve?jtbd=agent-regression-leaderboards"},{"id":"llm-as-judge-calibration","name":"LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation","shortName":"LLM-as-judge calibration","problemStatement":"Calibrate LLM-as-judge graders against human gold labels and deterministic assertions so your eval harness does not reward verbose, sycophantic, or self-preferring outputs.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","depotPickOverall":8.23,"depotPickAsOf":"2026-10-28","runnersUpIds":["patronus","ragas","galileo","confident-ai","promptfoo"],"whenNotTopPick":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","isNearTie":false,"scoreMargin":0.89,"pickConfidenceBand":"clear_leader","coPickProviderId":"patronus","url":"/jtbds/llm-as-judge-calibration","decideUrl":"/api/decide?jtbd=llm-as-judge-calibration","resolveUrl":"/api/resolve?jtbd=llm-as-judge-calibration"},{"id":"cost-latency-per-task-benchmarking","name":"Cost-per-Solved-Task, Token Efficiency & Latency Pareto Benchmarking","shortName":"Cost & latency per-task benchmarking","problemStatement":"Measure end-to-end USD cost, token burn (including retries, prompt cache hits, and sub-agent calls), and p50/p95 wall-clock latency per successfully solved task — so you optimize the cost-capability Pareto frontier instead of raw per-token list price.","depotPickProviderId":"langfuse","depotPickName":"Langfuse","depotPickOverall":8.66,"depotPickAsOf":"2026-10-28","runnersUpIds":["portkey","openrouter","braintrust","logfire"],"whenNotTopPick":"Gateway-level A/B routing, virtual keys, and budget caps across providers — Portkey; one-API multi-model price/latency comparison across 60+ hosts — OpenRouter; eval-first experiment diffs — Braintrust; OpenTelemetry Python app+LLM latency profiling — Pydantic Logfire.","isNearTie":true,"scoreMargin":0.35,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"portkey","url":"/jtbds/cost-latency-per-task-benchmarking","decideUrl":"/api/decide?jtbd=cost-latency-per-task-benchmarking","resolveUrl":"/api/resolve?jtbd=cost-latency-per-task-benchmarking"},{"id":"human-labeled-eval-review-queues","name":"Human-in-the-Loop Annotation Queues, Trace Labeling & Gold-Set Curation","shortName":"Human eval review queues","problemStatement":"Route low-confidence production traces, thumbs-down user sessions, and edge-case failures into structured human review queues so domain experts can label failure modes and promote curated examples into your golden eval dataset.","depotPickProviderId":"langsmith","depotPickName":"LangSmith","depotPickOverall":8.09,"depotPickAsOf":"2026-10-28","runnersUpIds":["langfuse","braintrust","arize-phoenix","honeyhive"],"whenNotTopPick":"Open-source / self-hosted annotation queues and dataset curation — Langfuse; eval-first human review + scorer alignment — Braintrust; OpenTelemetry OSS trace labeling — Arize Phoenix; rapid team trace curation — HoneyHive.","isNearTie":false,"scoreMargin":-0.51,"pickConfidenceBand":"moderate_lead","coPickProviderId":"langfuse","url":"/jtbds/human-labeled-eval-review-queues","decideUrl":"/api/decide?jtbd=human-labeled-eval-review-queues","resolveUrl":"/api/resolve?jtbd=human-labeled-eval-review-queues"}],"existingEvalJtbds":[{"id":"evals","name":"Prompt/agent eval harness","shortName":"Prompt/agent eval harness","problemStatement":"Regression-test agent quality in CI before ship.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","depotPickOverall":8.23,"depotPickAsOf":"2026-10-28","runnersUpIds":["langfuse","langsmith","arize-phoenix","confident-ai","langwatch","comet-opik","honeyhive","logfire"],"whenNotTopPick":"Budget OSS-only — Langfuse/Phoenix. Already standardized on LangSmith datasets. Humanloop is sunset (Anthropic acquihire) — do not pick.","isNearTie":false,"scoreMargin":-0.37,"pickConfidenceBand":"moderate_lead","coPickProviderId":"langfuse","url":"/jtbds/evals","decideUrl":"/api/decide?jtbd=evals","resolveUrl":"/api/resolve?jtbd=evals"},{"id":"oss-evals","name":"OSS / CI-native LLM evals","shortName":"OSS / CI-native LLM evals","problemStatement":"Run prompt evals and red teams in CI without paying for a hosted eval SaaS.","depotPickProviderId":"promptfoo","depotPickName":"Promptfoo","depotPickOverall":8.42,"depotPickAsOf":"2026-10-28","runnersUpIds":["ragas","braintrust","langsmith","confident-ai","comet-opik","langwatch"],"whenNotTopPick":"Need hosted collaboration/datasets for PMs — Braintrust; RAG-only metrics — Ragas.","isNearTie":true,"scoreMargin":0.14,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"ragas","url":"/jtbds/oss-evals","decideUrl":"/api/decide?jtbd=oss-evals","resolveUrl":"/api/resolve?jtbd=oss-evals"},{"id":"rag-evals","name":"RAG-specific evaluation metrics","shortName":"RAG-specific evaluation metrics","problemStatement":"Measure faithfulness, context precision/recall for retrieval pipelines.","depotPickProviderId":"ragas","depotPickName":"Ragas","depotPickOverall":8.33,"depotPickAsOf":"2026-10-28","runnersUpIds":["promptfoo","braintrust","galileo"],"whenNotTopPick":"General agent/tool evals — Promptfoo/Braintrust.","isNearTie":true,"scoreMargin":-0.14,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"promptfoo","url":"/jtbds/rag-evals","decideUrl":"/api/decide?jtbd=rag-evals","resolveUrl":"/api/resolve?jtbd=rag-evals"},{"id":"production-guardrail-evals","name":"Production guardrails & hallucination monitoring","shortName":"Production guardrails & hallucination mo…","problemStatement":"Detect and block bad outputs in production with research-grade metrics.","depotPickProviderId":"galileo","depotPickName":"Galileo","depotPickOverall":7.45,"depotPickAsOf":"2026-10-28","runnersUpIds":["braintrust","maxim","langsmith","arthur-ai","patronus","confident-ai","honeyhive"],"whenNotTopPick":"Pre-release simulation focus — Maxim; lightweight CI — Promptfoo.","isNearTie":false,"scoreMargin":-0.77,"pickConfidenceBand":"moderate_lead","coPickProviderId":"braintrust","url":"/jtbds/production-guardrail-evals","decideUrl":"/api/decide?jtbd=production-guardrail-evals","resolveUrl":"/api/resolve?jtbd=production-guardrail-evals"},{"id":"continuous-eval-ci","name":"Continuous eval gates in CI/CD","shortName":"Eval CI gates","problemStatement":"Block merges when agent prompts/tools regress — hosted eval suites, datasets, and pass/fail gates wired to GitHub/GitLab CI.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","depotPickOverall":8.23,"depotPickAsOf":"2026-10-28","runnersUpIds":["promptfoo","confident-ai","langsmith","patronus"],"whenNotTopPick":"Want OSS-first local gates (Promptfoo), DeepEval/Confident already standardized, or LangSmith is the org tracing system of record.","isNearTie":true,"scoreMargin":-0.12,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"promptfoo","url":"/jtbds/continuous-eval-ci","decideUrl":"/api/decide?jtbd=continuous-eval-ci","resolveUrl":"/api/resolve?jtbd=continuous-eval-ci"},{"id":"synthetic-eval-data","name":"Synthetic evaluation & scenario data","shortName":"Synthetic eval data","problemStatement":"Generate diverse agent test scenarios, adversarial prompts, and labeled trajectories without waiting on production traffic.","depotPickProviderId":"openai","depotPickName":"OpenAI","depotPickOverall":8.22,"depotPickAsOf":"2026-10-28","runnersUpIds":["anthropic","google-ai","fireworks","promptfoo"],"whenNotTopPick":"Need open-model bulk generation (Fireworks), or Promptfoo red-team generators already cover the matrix.","isNearTie":true,"scoreMargin":-0.21,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"anthropic","url":"/jtbds/synthetic-eval-data","decideUrl":"/api/decide?jtbd=synthetic-eval-data","resolveUrl":"/api/resolve?jtbd=synthetic-eval-data"},{"id":"adversarial-red-team-evals","name":"Adversarial red-team evaluation suites","shortName":"Red-team evals","problemStatement":"Continuously attack prompts/agents with jailbreak corpora and safety suites before and after each release.","depotPickProviderId":"promptfoo","depotPickName":"Promptfoo","depotPickOverall":8.42,"depotPickAsOf":"2026-10-28","runnersUpIds":["patronus","confident-ai","galileo","lakera"],"whenNotTopPick":"Patronus safety SKUs preferred, or Confident/DeepEval already embedded.","isNearTie":false,"scoreMargin":0.9,"pickConfidenceBand":"clear_leader","coPickProviderId":"patronus","url":"/jtbds/adversarial-red-team-evals","decideUrl":"/api/decide?jtbd=adversarial-red-team-evals","resolveUrl":"/api/resolve?jtbd=adversarial-red-team-evals"},{"id":"synthetic-persona-pre-flight","name":"Multi-Turn Synthetic User & Adversarial Persona Simulation","shortName":"Synthetic persona pre-flight","problemStatement":"Stress-test voice and chat agents against hundreds of realistic, frustrated or adversarial synthetic personas in multi-turn conversations before a production rollout, with scored transcripts as the release gate.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","depotPickOverall":8.23,"depotPickAsOf":"2026-10-28","runnersUpIds":["promptfoo","patronus","galileo"],"whenNotTopPick":"Open-source CLI red-teaming in CI — promptfoo; hallucination/judge models — Patronus; enterprise eval observability — Galileo.","isNearTie":true,"scoreMargin":-0.14,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"promptfoo","url":"/jtbds/synthetic-persona-pre-flight","decideUrl":"/api/decide?jtbd=synthetic-persona-pre-flight","resolveUrl":"/api/resolve?jtbd=synthetic-persona-pre-flight"}],"sourcedPricingPins":[{"providerId":"braintrust","providerName":"Braintrust","summary":"Starter $0/mo (1 GB processed data then $4/GB; 10k scores then $2.50/1k); Pro $249/mo (5 GB then $3/GB; 50k scores then $1.50/1k).","sourceUrl":"https://www.braintrust.dev/pricing","asOf":"2026-10-21"},{"providerId":"promptfoo","providerName":"Promptfoo","summary":"Open-source Apache-2.0 local eval & red-team CLI ($0 OSS library; no hosted quota included).","sourceUrl":"https://github.com/promptfoo/promptfoo/blob/main/LICENSE","asOf":"2026-10-25"},{"providerId":"langfuse","providerName":"Langfuse","summary":"MIT OSS self-host ($0) or Cloud Hobby $0 (50k units), Core $29/mo, Pro $199/mo (100k units included, graduated $8→$6 per 100k).","sourceUrl":"https://langfuse.com/pricing","asOf":"2026-10-21"},{"providerId":"langsmith","providerName":"LangSmith","summary":"Developer $0/mo (1 seat, 5,000 base traces included then $0.005/trace); Plus $39/seat/mo (10,000 base traces included).","sourceUrl":"https://www.langchain.com/pricing","asOf":"2026-10-22"},{"providerId":"e2b","providerName":"E2B","summary":"Hobby $0 base (up to 20 concurrent sandboxes) / Pro $150/mo (100 concurrent); compute $0.000014/vCPU-s and $0.0000045/GiB-s.","sourceUrl":"https://e2b.dev/pricing","asOf":"2026-10-28"},{"providerId":"browserbase","providerName":"Browserbase","summary":"Free $0 (1 browser hour, 3 concurrent); Developer $20/mo (100 hrs included, then $0.12/hr); Startup $99/mo (500 hrs, then $0.10/hr).","sourceUrl":"https://www.browserbase.com/pricing","asOf":"2026-10-28"},{"providerId":"galileo","providerName":"Galileo","summary":"Free $0/mo; Pro $100/mo on official pricing page.","sourceUrl":"https://galileo.ai/pricing","asOf":"2026-10-28"},{"providerId":"ragas","providerName":"Ragas","summary":"Open-source Apache-2.0 RAG evaluation toolkit ($0 OSS library).","sourceUrl":"https://raw.githubusercontent.com/vibrantlabsai/ragas/main/LICENSE","asOf":"2026-10-25"}],"selfEvalLoop":{"metrics":["passAt1 (single-trial task completion rate)","passAtKFloor (pass^3 or pass^5 consistency across repeated trials)","costPerSolvedTaskUsd (totalRunSpendUsd / verifiedSolvedCount — failed tasks never count as cheap successes)","p95LatencyMs (end-to-end wall-clock task latency including retries and tool calls)"],"judgeCalibrationRules":["Prefer deterministic outcome graders (unit tests, AST match, SQL/DOM state diff, Countersign receipt) over LLM prose scoring whenever possible.","When LLM-as-judge is required, use a cross-family judge model (e.g. grade an OpenAI agent with Anthropic or vice versa) to avoid self-preference bias.","Randomize candidate order (A/B vs B/A) and normalize length to neutralize position and verbosity bias.","Require ≥85% agreement against a 50–100 example human-labeled gold slice before enforcing a judge in CI."],"commands":["curl -sS 'https://getagentdepot.com/api/evals/plan?task=coding' | jq .","curl -sS 'https://getagentdepot.com/api/decide?jtbd=agent-trajectory-tool-use-evals' | jq '{pick: .pick.id, coPick: .nearTie.coPickProviderId, whenNot: .whenNotTopPick}'","curl -sS 'https://getagentdepot.com/api/benchmark/tasks?type=coding' | jq '.tasks | length'"]},"entrypoints":{"evalsHubUi":"/evals","evalsPlaybookUi":"/playbooks/agent-evals","evalsPlaybookJson":"/api/playbooks/agent-evals","evalsPlanApi":"/api/evals/plan?task={coding|browser|computer-use|tool-use|rag|multi-turn|terminal|ml|security}","depotBenchmarkSuiteUi":"/benchmark","depotBenchmarkSuiteApi":"/api/benchmark"},"honesty":{"providerCatalogFrozenAt":305,"noInventedBenchmarkScores":true,"noPaidPlacement":true,"agentSubmittedContentUntrusted":true,"note":"All picks come from the frozen 305-provider catalog. Public benchmarks link to their official leaderboards/repos with no fabricated scores. Community benchmark runs and insights are untrusted data and never mutate Depot editorial scores."}}