{"schemaVersion":"agentdepot.verification_hub.v1","asOf":"2026-10-28","stamp":"[⊙_⊙] ASSAY-VERIFIED","title":"Agent Outcome Verification, Countersign Protocol & Fail-Closed Promotion Gate","positioning":"Why 'Done' is a hypothesis until an external deterministic oracle checks it — and how Agent Provider Depot enforces fail-closed epistemic honesty across all 206 JTBDs.","clerkAssayStamps":[{"stamp":"[⊙_⊙] ASSAY: OUTCOME-VERIFIED","promotionClass":"verified_outcome_backed","confidenceBand":"±0.35 (high_verified)","rule":"Requires public verified outcome count n > 0 AND bound Countersign receipts passing scripts/promotion-predicate.mjs."},{"stamp":"[⊙_⊙] ASSAY: EDITORIAL PRIOR","promotionClass":"deterministic_source_reverified","confidenceBand":"±0.50 (medium_reverified)","rule":"Default honest label when pricing/sources are verified but public outcome count is n = 0. Cite line says 'editorial prior'."},{"stamp":"[⊙_⊙] ASSAY: FOOL'S GOLD / QUARANTINED","promotionClass":"stale_editorial_downgraded","confidenceBand":"±1.20 (provisional_stale)","rule":"Applied automatically when editorial review exceeds 30 days without re-verification or fails the promotion predicate."}],"countersignProtocol":{"name":"Countersign Outcome Verification Protocol","url":"https://countersign.signalry.ai","llmsTxt":"https://countersign.signalry.ai/llms.txt","siblingDisclosure":"Countersign is built by Signalry (sibling to Agent Provider Depot). Because of that relationship, Agent Depot runs behind an explicit Neutrality Firewall: scripts/promotion-predicate.mjs fails closed on any unbound or unverified receipt.","quickCommands":{"fileClaim":"cs claim \"Deployed v0.15.0 to production\" --check http_get:https://getagentdepot.com/api/health --check json_path:https://getagentdepot.com/api/health#$.status=ok","verifyReceipt":"cs verify cs-0dccf4b7","submitOutcomeToDepot":"curl -X POST https://getagentdepot.com/api/outcomes/verify -H \"Content-Type: application/json\" -d '{\"claimId\":\"cs-0dccf4b7\",\"jtbdId\":\"coding-agent\",\"providerId\":\"anthropic\",\"decisionStatus\":\"accepted\"}'"}},"quarantineRemediationCaseStudy":{"quarantineReceiptId":"qint_cc0b640320dea3e6","scoresMutated":false,"picksMutated":false,"summary":"In October 2026, independent auditor verification caught 11 of 20 frozen-cohort jobs (including postgres-vector) wearing verified_outcome_backed while public verified outcomes were n=0. We built scripts/promotion-predicate.mjs, wired it into npm run check:picks, and demoted all unearned labels to deterministic_source_reverified ('editorial prior') without mutating a single score or pick.","predicateScript":"scripts/promotion-predicate.mjs","contractDoc":"docs/quarantine-receipt.md"},"verificationJtbds":[{"id":"verifier-backed-self-correction-loops","name":"Verifier-Backed Runtime Self-Correction & Critic Loops (Compile / Test / Schema Gate)","shortName":"Verifier-backed self-correction loops","problemStatement":"Wrap agent generation steps in a bounded runtime verifier loop (compiler/typecheck, unit tests, JSON schema validator, or deterministic SQL/DOM check) so the agent repairs its own errors before returning an answer — without entering infinite retry spirals.","depotPickProviderId":"e2b","depotPickName":"E2B","overallScore":8.56,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["guardrails-ai","braintrust","browserbase"],"whenNotTopPick":"Need deterministic Pydantic/JSON-schema output validation and re-asking without full code execution — Guardrails AI; need offline + CI evaluation of whether self-correction steps actually improve pass@1 vs pass@k cost — Braintrust; need headless DOM/network verification for web actions — Browserbase.","decideUrl":"https://getagentdepot.com/api/decide?jtbd=verifier-backed-self-correction-loops","humanHref":"https://getagentdepot.com/jtbds/verifier-backed-self-correction-loops"},{"id":"agent-trajectory-tool-use-evals","name":"Agent Trajectory, Tool-Call & Multi-Step Plan Evaluation","shortName":"Agent trajectory & tool-use evals","problemStatement":"Grade an agent's multi-step execution trajectory — tool selection, JSON argument schemas, call ordering, state mutations, and retry recovery — instead of only scoring the final text string.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","overallScore":8.23,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["langsmith","arize-phoenix","langfuse","promptfoo"],"whenNotTopPick":"LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo.","decideUrl":"https://getagentdepot.com/api/decide?jtbd=agent-trajectory-tool-use-evals","humanHref":"https://getagentdepot.com/jtbds/agent-trajectory-tool-use-evals"},{"id":"llm-as-judge-calibration","name":"LLM-as-Judge Calibration, Rubric Alignment & Bias Mitigation","shortName":"LLM-as-judge calibration","problemStatement":"Calibrate LLM-as-judge graders against human gold labels and deterministic assertions so your eval harness does not reward verbose, sycophantic, or self-preferring outputs.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","overallScore":8.23,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["patronus","ragas","galileo","confident-ai","promptfoo"],"whenNotTopPick":"Dedicated regulated-industry hallucination & judge models (Lynx) — Patronus AI; research-grade RAG faithfulness/context metrics — Ragas; enterprise guardrail + hallucination scoring — Galileo; pytest-native G-Eval/DAG rubrics — Confident AI; OSS local assertion matrices — Promptfoo.","decideUrl":"https://getagentdepot.com/api/decide?jtbd=llm-as-judge-calibration","humanHref":"https://getagentdepot.com/jtbds/llm-as-judge-calibration"},{"id":"automated-red-teaming-self-hardening","name":"Continuous Adversarial Red-Teaming, Prompt-Injection & Policy Self-Hardening","shortName":"Automated red-teaming & self-hardening","problemStatement":"Continuously generate adversarial attacks (indirect prompt injection in untrusted tool outputs, PII exfiltration, unauthorized tool calls, jailbreaks) against your agent, patch system guardrails, and verify that attack success rate drops without spiking false-positive refusals.","depotPickProviderId":"promptfoo","depotPickName":"Promptfoo","overallScore":8.42,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["lakera","guardrails-ai","patronus"],"whenNotTopPick":"Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI.","decideUrl":"https://getagentdepot.com/api/decide?jtbd=automated-red-teaming-self-hardening","humanHref":"https://getagentdepot.com/jtbds/automated-red-teaming-self-hardening"},{"id":"agent-sandbox","name":"Secure code sandbox for agents","shortName":"Secure code sandbox for agents","problemStatement":"Execute untrusted LLM-generated code safely.","depotPickProviderId":"e2b","depotPickName":"E2B","overallScore":8.56,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["modal","daytona","fly-machines","upstash-box"],"whenNotTopPick":"Need persistent full IDE workspaces — Daytona. Python jobs that aren't untrusted REPL — Modal.","decideUrl":"https://getagentdepot.com/api/decide?jtbd=agent-sandbox","humanHref":"https://getagentdepot.com/jtbds/agent-sandbox"}],"links":{"self":"https://getagentdepot.com/api/verification","humanPage":"https://getagentdepot.com/verification","outcomesApi":"https://getagentdepot.com/api/outcomes","verifyIntakeApi":"https://getagentdepot.com/api/outcomes/verify","falsifyApi":"https://getagentdepot.com/api/falsify","otsProofsApi":"https://getagentdepot.com/api/pricing-history/ots"}}