{"schemaVersion":"agentdepot.eval_plan.v1","asOf":"2026-10-28","storesIpAddresses":false,"query":"terminal","matchMode":"matched_task","taskType":"terminal","label":"Terminal, Shell, DevOps & Sysadmin Agents","recommendedBenchmarks":[{"id":"terminal-bench","name":"Terminal-Bench","officialUrl":"https://tbench.ai/","graderType":"In-container shell verification script"},{"id":"swe-bench-verified","name":"SWE-bench Verified","officialUrl":"https://www.swebench.com/","graderType":"Container test suite (git apply + pytest)"}],"depotBenchmarkSuite":{"type":"coding","tasksApi":"/api/benchmark/tasks?type=coding","submitApi":"/api/benchmark/runs"},"stackPicks":{"evalHarness":{"jtbdId":"coding-agent-evals","pickProviderId":"e2b","coPickProviderId":"daytona","runnersUpIds":["daytona","modal"]},"sandboxOrRunner":{"jtbdId":"agent-sandbox","pickProviderId":"e2b","coPickProviderId":"modal","runnersUpIds":["modal","daytona"]},"ciGate":{"jtbdId":"continuous-eval-ci","pickProviderId":"braintrust","coPickProviderId":"promptfoo","runnersUpIds":["promptfoo","confident-ai"]},"judgeCalibration":{"jtbdId":"cost-latency-per-task-benchmarking","pickProviderId":"langfuse","coPickProviderId":"portkey","runnersUpIds":["portkey","openrouter"]}},"targetMetrics":["container verifier exit=0 pass@1","pass^5 idempotency","destructive-command safety=100%","costPerSolvedTaskUsd"],"antiContaminationRule":"Run each episode in an ephemeral Firecracker microVM with no host mounts and verify final filesystem/process state deterministically.","nextCalls":["/api/decide?jtbd=coding-agent-evals","/api/benchmark/tasks?type=coding","/api/playbooks/agent-evals"]}