{"schemaVersion":"agentdepot.learning_hub.v1","asOf":"2026-10-28","title":"Agent Learning, Memory Consolidation & Self-Improvement Hub","positioning":"How autonomous agents evaluate their own traces, consolidate episodic/semantic memory, optimize prompts and skills in sandboxes, and gate every self-modification with deterministic verifiers.","firstPrinciplesWarning":"Ungrounded LLM self-critique ('are you sure?') degrades accuracy and burns 2-4x tokens (Huang et al.). Every self-improvement loop MUST be anchored by an external deterministic verifier (compiler, unit test, schema check, or Countersign receipt).","sixStageLoop":[{"stage":"01 // CAPTURE & TRACE","name":"Deterministic Trace & Outcome Capture","summary":"Record tool calls, arguments, latencies, token spend, and terminal outcomes with OpenTelemetry spans so every failure is reproducible.","recommendedJtbd":"agent-trajectory-tool-use-evals","keyRule":"Never optimize what you cannot replay deterministically."},{"stage":"02 // GRADE & FALSIFY","name":"External Verifier & Promotion Predicate Check","summary":"Grade trajectories with an external oracle (compiler, unit test, schema check, or Countersign receipt) rather than uncalibrated LLM self-praise.","recommendedJtbd":"verifier-backed-self-correction-loops","keyRule":"Separate verified_outcome_backed from deterministic_source_reverified (editorial prior)."},{"stage":"03 // REFLECT & CLUSTER","name":"Failure Trace Clustering & Root-Cause Critique","summary":"Cluster failed trajectories by root cause (tool schema drift, context rot, stale SKU, prompt injection, or overclaimed confidence).","recommendedJtbd":"failure-trace-reflection-rule-synthesis","keyRule":"Name the exact violated invariant before drafting a fix."},{"stage":"04 // SYNTHESIZE FIX","name":"Rule, Skill, Prompt, or Adapter Synthesis","summary":"Synthesize the minimal durable fix: a guardrail rule, a unit-tested sandbox skill (SKILL.md), a DSPy-compiled prompt, or a DPO/LoRA adapter.","recommendedJtbd":"prompt-program-self-optimization","keyRule":"Prefer a deterministic script gate over adding another paragraph of prompt instructions."},{"stage":"05 // REPLAY & GATE","name":"Held-Out Regression & Adversarial Replay Gate","summary":"Run the candidate fix against held-out golden tasks and adversarial red-team probes in CI. Reject any fix with regressionCount > 0.","recommendedJtbd":"automated-red-teaming-self-hardening","keyRule":"Require regressionCount === 0 on held-out tasks before promotion."},{"stage":"06 // CRYSTALLIZE & WITNESS","name":"Skill Library Promotion & Countersign Receipt","summary":"Commit the passing skill/gate into version control, log the cycle in data/self-improvement-log.json, and anchor the release with a Countersign receipt.","recommendedJtbd":"skill-library-crystallization","keyRule":"File a verifiable receipt (cs claim) and log public corrections openly."}],"learningJtbdsCount":8,"learningJtbds":[{"id":"agent-episodic-semantic-memory-learning","name":"Agent Episodic & Semantic Memory Consolidation (Cross-Session Learning)","shortName":"Episodic & semantic memory learning","problemStatement":"Extract durable user preferences, domain facts, and past episode outcomes from raw conversation and tool traces so an agent learns across sessions without bloating the context window or poisoning future runs with stale hallucinations.","depotPickProviderId":"mem0","depotPickName":"Mem0","depotPickOverallScore":7.59,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["zep","letta","supabase-pgvector"],"whenNotTopPick":"Need temporal bi-temporal knowledge-graph edges with explicit validity windows (`valid_from` / `invalid_at`) for changing facts — Zep (Graphiti); need explicit OS-style self-editing core/archival memory blocks managed by the agent via tool calls — Letta; need DIY SQL + vector episodic tables inside existing Postgres — Supabase pgvector.","decideEndpoint":"https://getagentdepot.com/api/decide?jtbd=agent-episodic-semantic-memory-learning","humanHref":"https://getagentdepot.com/jtbds/agent-episodic-semantic-memory-learning"},{"id":"prompt-program-self-optimization","name":"Automated Prompt, Few-Shot & Agent Program Optimization (DSPy / MIPROv2 / TextGrad)","shortName":"Prompt & agent program optimization","problemStatement":"Compile and optimize system instructions, tool descriptions, and multi-hop few-shot demonstrations against a deterministic metric and golden dataset — replacing manual prompt trial-and-error with versioned, eval-gated prompt compilation.","depotPickProviderId":"braintrust","depotPickName":"Braintrust","depotPickOverallScore":8.23,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["promptfoo","langsmith","langfuse"],"whenNotTopPick":"Local CLI / YAML matrix optimization and adversarial red-teaming in CI with zero SaaS dependency — Promptfoo; LangGraph node-level prompt versioning and hub playgrounds — LangSmith; open-source MIT prompt management + dataset experiment linkage — Langfuse.","decideEndpoint":"https://getagentdepot.com/api/decide?jtbd=prompt-program-self-optimization","humanHref":"https://getagentdepot.com/jtbds/prompt-program-self-optimization"},{"id":"failure-trace-reflection-rule-synthesis","name":"Failure Trace Clustering, Reflexion & Guardrail Rule Synthesis","shortName":"Failure trace reflection & rule synthesis","problemStatement":"Automatically cluster production agent failure spans (tool schema errors, retry loops, judge rejections, user thumbs-down), run root-cause reflection, and synthesize candidate routing rules or negative constraints into aregression dataset.","depotPickProviderId":"langsmith","depotPickName":"LangSmith","depotPickOverallScore":8.09,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["langfuse","braintrust","arize-phoenix"],"whenNotTopPick":"Open-source MIT self-hosted trace clustering, cost attribution, and dataset promotion — Langfuse; CI-first eval diffing and scorer feedback loops — Braintrust; OpenTelemetry-native span clustering and drift analysis — Arize Phoenix.","decideEndpoint":"https://getagentdepot.com/api/decide?jtbd=failure-trace-reflection-rule-synthesis","humanHref":"https://getagentdepot.com/jtbds/failure-trace-reflection-rule-synthesis"},{"id":"skill-library-crystallization","name":"Executable Skill & Tool Library Crystallization (Voyager-Style Verified Code Reuse)","shortName":"Skill library crystallization","problemStatement":"Turn multi-step trajectories that succeeded in an isolated sandbox into parameterized, unit-tested, reusable skills (`SKILL.md` + deterministic scripts) so future agent runs invoke a verified tool instead of re-deriving brittle multi-turn plans.","depotPickProviderId":"e2b","depotPickName":"E2B","depotPickOverallScore":8.56,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["letta","braintrust","daytona"],"whenNotTopPick":"Need stateful, persistent agent memory blocks attached to an agent runtime — Letta; need the evaluation harness to grade candidate skills across 100+ test inputs before promotion — Braintrust; need full git-workspace container persistence across days — Daytona.","decideEndpoint":"https://getagentdepot.com/api/decide?jtbd=skill-library-crystallization","humanHref":"https://getagentdepot.com/jtbds/skill-library-crystallization"},{"id":"preference-dpo-sft-trace-distillation","name":"Trajectory SFT, DPO & GRPO Distillation (Teacher-to-Student Agent Fine-Tuning)","shortName":"Trajectory SFT / DPO / GRPO distillation","problemStatement":"Distill verified high-scoring trajectories from an expensive frontier teacher model (plus rejected failure pairs) into a smaller, faster open-weight student model via LoRA SFT, Direct Preference Optimization (DPO), or Verifier-Guided GRPO.","depotPickProviderId":"together","depotPickName":"Together AI","depotPickOverallScore":8.2,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["fireworks","weights-biases","modal"],"whenNotTopPick":"Ultra-low-latency serverless LoRA multi-tenant deployment after tuning — Fireworks AI; experiment tracking, dataset lineage, and reward-curve telemetry across RL/DPO sweeps — Weights & Biases; raw multi-node H100/B200 clusters for custom Megatron/vLLM RL loops — Lambda or Modal.","decideEndpoint":"https://getagentdepot.com/api/decide?jtbd=preference-dpo-sft-trace-distillation","humanHref":"https://getagentdepot.com/jtbds/preference-dpo-sft-trace-distillation"},{"id":"online-bandit-model-routing-adaptation","name":"Online Bandit & Pareto Model Routing Adaptation (Cost / Latency / Quality Learning)","shortName":"Online bandit model routing adaptation","problemStatement":"Continuously adapt model and provider routing weights per task class using live latency, error-rate, and verifier pass-rate telemetry — routing routine steps to fast/cheap models while escalating hard steps to frontier reasoning models.","depotPickProviderId":"openrouter","depotPickName":"OpenRouter","depotPickOverallScore":8,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["portkey","helicone","braintrust"],"whenNotTopPick":"Enterprise AI gateway with semantic caching, guardrails, and virtual keys across teams — Portkey; observability-first proxy with header-based caching and cost analytics — Helicone; offline dataset Pareto scoring to set initial routing priors — Braintrust.","decideEndpoint":"https://getagentdepot.com/api/decide?jtbd=online-bandit-model-routing-adaptation","humanHref":"https://getagentdepot.com/jtbds/online-bandit-model-routing-adaptation"},{"id":"automated-red-teaming-self-hardening","name":"Continuous Adversarial Red-Teaming, Prompt-Injection & Policy Self-Hardening","shortName":"Automated red-teaming & self-hardening","problemStatement":"Continuously generate adversarial attacks (indirect prompt injection in untrusted tool outputs, PII exfiltration, unauthorized tool calls, jailbreaks) against your agent, patch system guardrails, and verify that attack success rate drops without spiking false-positive refusals.","depotPickProviderId":"promptfoo","depotPickName":"Promptfoo","depotPickOverallScore":8.42,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["lakera","guardrails-ai","patronus"],"whenNotTopPick":"Low-latency inline runtime prompt-injection firewall in production — Lakera Guard; programmable input/output validator rails in Python/TS — Guardrails AI; managed enterprise hallucination and policy evaluation suites — Patronus AI.","decideEndpoint":"https://getagentdepot.com/api/decide?jtbd=automated-red-teaming-self-hardening","humanHref":"https://getagentdepot.com/jtbds/automated-red-teaming-self-hardening"},{"id":"verifier-backed-self-correction-loops","name":"Verifier-Backed Runtime Self-Correction & Critic Loops (Compile / Test / Schema Gate)","shortName":"Verifier-backed self-correction loops","problemStatement":"Wrap agent generation steps in a bounded runtime verifier loop (compiler/typecheck, unit tests, JSON schema validator, or deterministic SQL/DOM check) so the agent repairs its own errors before returning an answer — without entering infinite retry spirals.","depotPickProviderId":"e2b","depotPickName":"E2B","depotPickOverallScore":8.56,"scoreAsOf":"2026-10-28","evidenceBasis":"editorial prior (deterministic_source_reverified, ±0.5)","runnersUpIds":["guardrails-ai","braintrust","browserbase"],"whenNotTopPick":"Need deterministic Pydantic/JSON-schema output validation and re-asking without full code execution — Guardrails AI; need offline + CI evaluation of whether self-correction steps actually improve pass@1 vs pass@k cost — Braintrust; need headless DOM/network verification for web actions — Browserbase.","decideEndpoint":"https://getagentdepot.com/api/decide?jtbd=verifier-backed-self-correction-loops","humanHref":"https://getagentdepot.com/jtbds/verifier-backed-self-correction-loops"}],"frameworksAndBenchmarks":[{"id":"dspy-miprov2","name":"DSPy & MIPROv2 (Stanford NLP)","category":"Prompt & Program Compilation","url":"https://dspy.ai","repoUrl":"https://github.com/stanfordnlp/dspy","whenToUse":"When manual prompt tweaking plateaus and you have >= 30 labeled examples + a deterministic metric to compile instructions and few-shot demos.","keyMechanism":"Joint Bayesian optimization over instructions and bootstrapped few-shot demonstrations (MIPROv2)."},{"id":"textgrad","name":"TextGrad (Zou Lab / Stanford)","category":"Textual Backpropagation","url":"https://github.com/zou-group/textgrad","repoUrl":"https://github.com/zou-group/textgrad","whenToUse":"When optimizing compound multi-step agent pipelines or code/molecule/prompt artifacts using structured textual gradients from a critic.","keyMechanism":"Backpropagates natural-language critiques through computation graphs of LLM calls."},{"id":"reflexion","name":"Reflexion (Shinn et al.)","category":"Verbal Reinforcement & Episodic Critique","url":"https://arxiv.org/abs/2303.11366","repoUrl":"https://github.com/noahshinn/reflexion","whenToUse":"When an agent fails a coding, reasoning, or tool-use episode and has a concrete environment error signal to reflect on before retrying.","keyMechanism":"Converts binary/scalar environment feedback into a structured verbal critique stored in episodic memory."},{"id":"voyager","name":"Voyager Skill Library Pattern (Wang et al.)","category":"Executable Skill Crystallization","url":"https://voyager.minedojo.org","repoUrl":"https://github.com/MineDojo/Voyager","whenToUse":"When your agent repeatedly solves similar multi-step tasks and should crystallize verified code into a reusable tool/skill library.","keyMechanism":"Iterative code synthesis inside a sandbox -> automated verification -> indexed skill library retrieval."},{"id":"grpo-dpo-star","name":"STaR / ReST^EM / DPO / Verifier-Guided GRPO","category":"Trajectory Distillation & RL","url":"https://arxiv.org/abs/2501.12948","repoUrl":"https://github.com/huggingface/trl","whenToUse":"When you have 1,000+ verifier-filtered trajectories from a frontier teacher model and want to distill them into a fast open-weight student.","keyMechanism":"Filters rollouts via deterministic verifiers (tests/math/schemas) and optimizes policy weights via DPO or Group Relative Policy Optimization."},{"id":"swe-agent-openhands-inspect","name":"SWE-agent / OpenHands / UK AISI Inspect AI","category":"Agent Evaluation & Sandboxed Harnesses","url":"https://inspect.ai-safety-institute.org.uk","repoUrl":"https://github.com/UKGovernmentBEIS/inspect_ai","whenToUse":"When benchmarking coding, terminal, and tool-use agents before and after a self-improvement change to prove non-regression.","keyMechanism":"Sandboxed execution harnesses paired with reproducible scorers on SWE-bench Verified, Terminal-Bench, and GAIA."},{"id":"longmemeval-locomo","name":"LongMemEval & LOCOMO","category":"Cross-Session Memory Benchmarks","url":"https://github.com/xiaowu0162/LongMemEval","repoUrl":"https://github.com/snap-research/locomo","whenToUse":"When testing whether your episodic/semantic memory layer (Mem0, Zep, Letta, pgvector) retrieves updated facts without temporal confusion.","keyMechanism":"Evaluates knowledge updates, temporal reasoning, multi-session synthesis, and abstention over long interaction histories."},{"id":"countersign-witness","name":"Countersign Outcome Verification Protocol","category":"Cryptographic & Public-API Witnessing","url":"https://countersign.signalry.ai","repoUrl":"https://countersign.signalry.ai/llms.txt","whenToUse":"When an agent finishes a deploy, audit, or self-improvement cycle and needs an independent verification receipt (cs-xxxx) rather than self-attestation.","keyMechanism":"Runs deterministic HTTP/DNS/TLS/JSON/git checks and issues signed receipts with counts_as_verified gating."}],"liveSelfImprovementLedger":{"description":"Agent Provider Depot runs this exact 6-stage loop on itself. Below are our live self-improvement cycles (including the qint_cc0b640320dea3e6 fail-closed promotion gate remediation).","cyclesCount":8,"cycles":[{"cycleId":"SIL-2026-10-28-001","date":"2026-10-28","stage1_capture":{"source":"Independent auditor quarantine receipt qint_cc0b640320dea3e6 +","symptom":"promotion_coherence_overclaim","observedDefect":"11 of 20 frozen-cohort jobs (including postgres-vector -> supabase-pgvector) and 36 governance pickAudits wore promotionClass=verified_outcome_backed / high_verified (±0.35) while public verified outcome count n=0 and cited cs-xxxx receipts were unbound repo/deploy claims."},"stage2_grade":{"verifier":"scripts/promotion-predicate.mjs","preFixDefectsCount":47,"postFixDefectsCount":0},"stage3_reflect":{"rootCause":"Promotion gate checked only whether countersignReceiptIds was non-empty in governance.json, without verifying that public outcome count n > 0 in data/insights.json or that cited cs-xxxx claims explicitly bound the providerId and jtbdId.","violatedPrinciple":"Commandment 02 (Thou Shalt Not Grade Thine Own Homework) — never wear verified_outcome_backed when external outcome count is zero."},"stage4_synthesize":{"artifactType":"predicate_gate_and_demotion","filesCreatedOrUpdated":["scripts/promotion-predicate.mjs","scripts/check-pick-authority.mjs","docs/quarantine-receipt.md","data/promotion-coherence-report.json","data/governance.json","data/audit-cohort.json","src/lib/decide.ts","src/lib/citation.ts"],"scoresMutated":false,"picksMutated":false},"stage5_replayGate":{"command":"npm run check:picks && node scripts/promotion-predicate.mjs","testsPassed":3,"regressionCount":0},"stage6_crystallize":{"quarantineReceiptId":"qint_cc0b640320dea3e6","status":"crystallized_in_ci"}},{"cycleId":"SIL-2026-10-28-002","date":"2026-10-28","stage1_capture":{"source":"first-party-agent independent live-site audit on v0.13.8 (5 defects reported)","symptom":"truncation_and_prose_hygiene_drift","observedDefect":"Mid-word ellipsis truncation on /api/decide pricingSummary, trailing comma in citation, raw underscore in UI scoreKind badge, and missing pricing unit match on new eval JTBDs."},"stage2_grade":{"verifier":"scripts/check-limits.mjs + scripts/check-data-correctness.mjs","preFixDefectsCount":5,"postFixDefectsCount":0},"stage3_reflect":{"rootCause":"Hard 280-char slice cut words mid-token instead of snapping to a word boundary, and new JTBDs needed explicit SKU unit coverage checks.","violatedPrinciple":"Machine & human legibility parity."},"stage4_synthesize":{"artifactType":"code_and_data_guardrail","filesCreatedOrUpdated":["src/lib/decide.ts","src/components/JobBrief.tsx","scripts/check-limits.mjs"],"scoresMutated":false,"picksMutated":false},"stage5_replayGate":{"command":"npm run check:limits && npm run check:data","testsPassed":2079,"regressionCount":0},"stage6_crystallize":{"quarantineReceiptId":"cs-a8c3611b","status":"crystallized_in_ci"}},{"cycleId":"SIL-2026-10-28-003","date":"2026-10-28","stage1_capture":{"source":"Self-improvement audit script (scripts/self-improve-audit.mjs)","symptom":"missing_learning_jtbd_coverage","observedDefect":"Agents asking how to consolidate episodic memory, compile prompts with DSPy/MIPROv2, cluster failure traces, crystallize skills in sandboxes, distill traces via DPO/GRPO, or run verifier-backed self-correction loops had no dedicated JTBDs."},"stage2_grade":{"verifier":"scripts/self-improve-audit.mjs","preFixDefectsCount":8,"postFixDefectsCount":0},"stage3_reflect":{"rootCause":"Catalog covered observability and evals (16 JTBDs) but lacked the closed-loop post-eval learning and self-improvement jobs (8 JTBDs).","violatedPrinciple":"Every evaluation failure should map to a concrete remediation and learning job."},"stage4_synthesize":{"artifactType":"catalog_and_playbook_expansion","filesCreatedOrUpdated":["scripts/migrations/v0150-learning-jtbds.mjs","src/lib/agentLearning.ts","src/app/learning/page.tsx","src/app/playbooks/agent-self-improvement/page.tsx"],"scoresMutated":false,"picksMutated":false},"stage5_replayGate":{"command":"npm run validate-data && npm run check:self-improve","testsPassed":12,"regressionCount":0},"stage6_crystallize":{"quarantineReceiptId":"qint_cc0b640320dea3e6","status":"crystallized_in_ci"}},{"cycleId":"SIL-2026-10-28-004","date":"2026-10-28","stage1_capture":{"source":"ChatGPT external decision-correctness & usability audit (, 21 concrete findings with exact reproduction URLs)","symptom":"shorthand_volume_silent_fallback_and_surface_inconsistency","observedDefect":"volume=100k silently fell back to 1,000 tasks ($1.03) and returned autoActOk: true under maxMonthly=10; compact next links dropped volume/maxMonthly; /api/resolve had gpt-5.4 vs gpt-6.1-sol mismatch and lacked advisoryDefaultOnly on unsupported constraints; /benchmark/runs/run_verified_depot_router_v0130 threw 500; /benchmark/leaderboard mixed officialEligible:false smoke runs into Authority Standing; and /api/blueprint?archetype=coding-agent-stack&budget=0 replaced the archetype instead of substituting $0 slots."},"stage2_grade":{"verifier":"scripts/self-improve-audit.mjs","preFixDefectsCount":21,"postFixDefectsCount":0},"stage3_reflect":{"rootCause":"Number(raw) in parseContext rejected shorthand suffixes (100k, 2.5k, 1m) and fell back to null (triggering the 1,000-task starter quote in actionableForJob), while continuation URLs, blueprint budget=0 substitution, and benchmark run detail templates lacked strict invariant checks.","violatedPrinciple":"Fail-closed spend gating — unparseable or shorthand workload parameters must never silently fall back to a smaller default volume and approve spend."},"stage4_synthesize":{"artifactType":"decision_correctness_and_surface_hardening","filesCreatedOrUpdated":["src/lib/context.ts","src/lib/actionable.ts","src/lib/answer.ts","src/lib/fleetV146.ts","src/lib/bootstrap.ts","src/app/api/resolve/route.ts","src/lib/masterpiece.ts","src/lib/benchmark/store.ts","src/app/benchmark/runs/[id]/page.tsx","src/app/benchmark/leaderboard/page.tsx","src/app/benchmark/tasks/[id]/page.tsx","src/app/jtbds/page.tsx","src/components/JobBrief.tsx","src/app/jtbds/[slug]/vs/[other]/page.tsx","src/components/RankTable.tsx","scripts/self-improve-audit.mjs"],"scoresMutated":false,"picksMutated":false},"stage5_replayGate":{"command":"npm run check:self-improve && npm test","testsPassed":54,"regressionCount":0},"stage6_crystallize":{"quarantineReceiptId":"qint_cc0b640320dea3e6","status":"crystallized_in_ci"}},{"cycleId":"SIL-2026-10-29-001","date":"2026-10-29","stage1_capture":{"source":"Scheduled self-improvement run (directory-agent) + data/history/pending-reprice.json + prior run notes","symptom":"stale_stored_prices_and_pending_list_disagreeing_with_newer_reviewed_reading","observedDefect":"7 stored prices no longer matched their official pages (Chutes Private Chute $1.80/h + $5.40 fee vs page $2.25/h + $6.75; Verda on-demand GPU rows vs a page that had moved twice). After a reviewed re-price from a newer reading, pending-reprice would still have listed the Verda rows against the older 2026-10-28 watch observation. Separately, check:data HTTP mode failed 18 /api/free pins on production because the v0.15.2 public scrub (correctly) removes approval provenance those pins quoted."},"stage2_grade":{"verifier":"scripts/check-data-correctness.mjs (offline + HTTP)","preFixDefectsCount":25,"postFixDefectsCount":0},"stage3_reflect":{"rootCause":"The derived pending list compared only the latest watch observation to the catalog and ignored the date of a newer reviewed re-price; the HTTP gate pinned internal text instead of proving public text = scrub(stored).","violatedPrinciple":"Machine-readable outputs must not be confidently wrong: a newer reviewed official reading supersedes an older observation (kept visible, never dropped), and gates must verify the public view without requiring private provenance."},"stage4_synthesize":{"artifactType":"reviewed_reprice_and_derived_view_rule","filesCreatedOrUpdated":["data/pricing-rows.json","data/price-reviews.json","data/corrections-log.json","data/history/*","scripts/lib/history.mjs","src/lib/history.ts","scripts/check-data-correctness.mjs"],"scoresMutated":false,"picksMutated":false},"stage5_replayGate":{"command":"validate-data, check:data (offline 380/380, HTTP 492/492 preview + production), check:limits, check:docs, check:matcher, check:picks, check:security, check:self-improve, npm test, build, check:deploy","testsPassed":122,"regressionCount":0},"stage6_crystallize":{"rule":"pending-reprice skips a row only when its asOf is later than the watch observation AND an 'updated' price-review on that date matches the catalog; the older observation is listed in supersededObservations. /api/free HTTP pins run only on strings proven equal to scrubPublicText(stored).","status":"crystallized_in_ci"}},{"cycleId":"SIL-2026-10-29-002","date":"2026-10-29","stage1_capture":{"source":"Scheduled self-improvement run (directory-agent): audit of /api/answer?q=cheapest+embeddings on production 0.16.3","symptom":"route_vs_quote_model_family_mismatch","observedDefect":"The embeddings pick Cohere was quoted at $0.30/1M input + $0.60/1M output (1,000-task estimate $2.07) from the legacy Command-light chat row; 9 job/provider workload quotes on embeddings, batch-cheap-embeddings, late-interaction-retrieval and rag-enterprise bound chat, GPU-hour or legacy SKUs (cohere x4, openai x2, google-ai x2, huggingface-inference x1)."},"stage2_grade":{"verifier":"scripts/test-fleet-backlog.mjs (every job x pick/runner-up workload quote) + /tmp audit of buildRecommendedWorkloadQuote","preFixDefectsCount":9,"postFixDefectsCount":0},"stage3_reflect":{"rootCause":"buildRecommendedWorkloadQuote kept every provider row on embedding jobs (it only stopped dropping embedding rows), and its legacy filter read only lifecycle == legacy, which the Cohere legacy rows do not set (they carry normalizationReason legacy_sku_not_current_matrix and '(legacy)' in the SKU).","violatedPrinciple":"A quote must price the same model family the job runs; no sourced price in that family means price null with a reason, never a neighbouring SKU. Same class as qint_8072152a0531f311 (rerank)."},"stage4_synthesize":{"artifactType":"quote_family_rule","filesCreatedOrUpdated":["src/lib/fleetV146.ts","src/lib/runtime-routes.ts","scripts/test-fleet-backlog.mjs","scripts/check-docs.mjs","data/corrections-log.json"],"scoresMutated":false,"picksMutated":false},"stage5_replayGate":{"command":"validate-data, check:data (380/380 offline, 492/492 HTTP preview), check:limits (2079/2079, 2194/2194 HTTP), check:docs (81/81, 100/100 HTTP), check:matcher 387/387, check:picks 471/471, check:security 91/91, check:self-improve 196/196 + 289/289 live preview, check:leaks, check:links, check:agent-native 356/356, npm test 139/139, build, check:deploy","testsPassed":139,"regressionCount":0},"stage6_crystallize":{"rule":"Embedding-model jobs (embeddings, batch-cheap-embeddings, late-interaction-retrieval) quote only embedding SKUs (isEmbeddingSkuRow) with no output-token leg, else null; no workload quote or runtime route binds a legacy SKU (isLegacySkuRow). Regression test asserts this for every job x pick/runner-up.","status":"crystallized_in_ci"}},{"cycleId":"SIL-2026-10-29-003","date":"2026-10-29","stage1_capture":{"source":"Scheduled self-improvement run (directory-agent): production /api/stats scorecard30d and the 12 PM ET check:deploy output","symptom":"status_ignores_targets_and_first_party_traffic_misbucketed","observedDefect":"scorecard30d.status read 'on_track' on day 1 while organicDiscoverabilityTop10Of20 was 0 against a day-1 target of 1; smoke.sh and the Node check scripts (bare curl / default node user agent) put our own deploy and preview check traffic into unverifiedAnonymousCallsPerWeek (86 -> 425/wk across one deploy check), and three MCP probes initialized with clientInfo names (cs, check, make-server-json) that classify as verified_external."},"stage2_grade":{"verifier":"scripts/test-fleet-backlog.mjs (v0.16.5 scorecard + first-party traffic tests) and check:deploy step (d) probes for the first-party check user agent","preFixDefectsCount":2,"postFixDefectsCount":0},"stage3_reflect":{"rootCause":"buildScorecard30d derived status from verified external calls alone; the check scripts never declared the first-party user agent the server already recognizes (FIRST_PARTY_UA), and probe clientInfo names were short non-generic strings.","violatedPrinciple":"A machine-readable status must agree with every number it summarizes; our own traffic must never be counted as outside use, and a first-party marker may only ever deflate, never inflate, verified external counts."},"stage4_synthesize":{"artifactType":"status_rule_and_first_party_tagging","filesCreatedOrUpdated":["src/lib/scorecard30d.ts","scripts/lib/first-party-ua.mjs","scripts/smoke.sh","scripts/check-deploy.mjs","scripts/probes/mcp.mjs","scripts/check-agent-native.mjs","scripts/make-server-json.mjs","scripts/check-*.mjs","scripts/self-improve-audit.mjs","scripts/test-fleet-backlog.mjs","data/corrections-log.json"],"scoresMutated":false,"picksMutated":false},"stage5_replayGate":{"command":"validate-data, check:status-counts, lint, check:data (380/380, 492/492 HTTP preview), check:limits (2079/2079, 2194/2194 HTTP), check:docs (100/100 HTTP), check:matcher 387/387, check:picks 471/471, check:security 91/91, check:self-improve 199/199 + 292/292 live preview, check:agent-native 356/356, check:leaks 272/0, check:links 214/0, smoke 113/113 preview, npm test 141/141, build, check:deploy (smoke 114/114)","testsPassed":141,"regressionCount":0},"stage6_crystallize":{"rule":"scorecard30d.status is 'on_track' only when every day-N target is met (unverifiedRankedPicksCount at_most, every other target at_least), else 'behind' with behindTargets and per-target targetStatus. Every first-party check script and smoke request sends the first-party user agent AgentProviderDepot-check/1 (scripts/lib/first-party-ua.mjs, SMOKE_UA) and MCP probes use first-party clientInfo names; the first-party user agent can only classify as first_party, never verified_external. Regression tests and check:deploy probes assert both.","status":"crystallized_in_ci"}},{"cycleId":"SIL-2026-10-30-001","date":"2026-10-30","stage1_capture":{"source":"Scheduled self-improvement run (directory-agent): audit of /api/stats scorecard30d and /api/preflight against the uncommitted 2026-10-30 discoverability baseline","symptom":"freshness_gates_pinned_to_fixed_date_and_discoverability_field_mismatch","observedDefect":"buildPreflightCheck computed price age with ageDays(asOf, '2026-10-28') on the provider's first row, so priceVerifiedWithinDays never grew and a stale price could not fail the 14-day gate or callEstimate.autoActOk; buildScorecard30d measured rankedPricesWithin14DaysPct against the same fixed date (would read 100% on 2026-11-11 when only 340/920 rows are <=14d old); readLatestDiscoverability read {surfaced, bestRank} while scripts/discoverability-check.mjs writes {appears, rank}, so a real top-10 hit counted 0."},"stage2_grade":{"verifier":"scripts/test-fleet-backlog.mjs v0.16.9 test (mocked clock 2026-12-31; synthetic discoverability rows; every committed discoverability file uses the script shape and its summary matches its rows)","preFixDefectsCount":3,"postFixDefectsCount":0},"stage3_reflect":{"rootCause":"Release-day constants (2026-10-28) were used as the 'now' for live freshness gates, and the scorecard reader was written against a different schema than the writer.","violatedPrinciple":"A freshness gate must measure age at request time on the exact row it authorizes; a metric reader must be tested against its writer's real output."},"stage4_synthesize":{"artifactType":"freshness_gate_and_schema_contract","filesCreatedOrUpdated":["src/lib/masterpiece.ts","src/lib/scorecard30d.ts","scripts/test-fleet-backlog.mjs","data/discoverability/2026-10-30.json"],"scoresMutated":false,"picksMutated":false},"stage5_replayGate":{"command":"validate-data, check:data 380/380, check:limits 2079/2079, check:docs 81/81, check:matcher 387/387, check:picks 471/471, check:security 91/91, check:self-improve 202/202, check:brand 151/151, npm test 159/159, build, check:deploy (smoke 114/114)","testsPassed":159,"regressionCount":0},"stage6_crystallize":{"rule":"Live price-age gates (preflight actionableGate and callEstimate.autoActOk, scorecard rankedPricesWithin14DaysPct) measure age at request/scorecard date via todayIso()/nowIso, on the row that is costed or quoted; per-call autoActOk also requires the quoted SKU <=14d. The scorecard reads discoverability via countDiscoverability(), which accepts the script's {appears, rank}; a test asserts every committed discoverability file uses that shape.","status":"crystallized_in_ci"}}]},"links":{"self":"https://getagentdepot.com/api/learning","humanPage":"https://getagentdepot.com/learning","diagnoseEndpoint":"https://getagentdepot.com/api/learning/diagnose?symptom=hallucinating-past-sessions","playbookPage":"https://getagentdepot.com/playbooks/agent-self-improvement","playbookMarkdown":"https://getagentdepot.com/playbooks/agent-self-improvement.md","playbookApi":"https://getagentdepot.com/api/playbooks/agent-self-improvement","evalsHub":"https://getagentdepot.com/evals","ethosHub":"https://getagentdepot.com/ethos"},"identifyHint":"Send X-Agent-Name: <your-agent> on each call (MCP: clientInfo.name) to be counted as a verified external agent."}