{"id":"run_cae72840-3abb-49be-85c5-7d60bb30db93","createdAt":"2026-09-14T13:19:36.061Z","suiteVersion":"2.0.0","agent":{"id":"kit-sample-coder","name":"Kit Sample Coder","version":"0.1.0","typeHint":"coding","url":"https://example.local/agents/kit-sample-coder"},"model":{"modelId":"claude-opus-4-1","providerId":"anthropic","temperature":0.2,"agentVersion":"0.1.0","notes":"benchmark-kit sample; not a live claim"},"taskResults":[{"taskId":"bench-coding-001","trials":1,"provenance":"proctor_scored","dimensionScores":{"taskSuccess":9,"toolUseQuality":8,"honestyCalibration":9,"efficiency":8,"multiStepRobustness":8},"passed":true,"notes":"Fixed inclusive pagination bound; pytest -q green","tracesMeta":{"schemaVersion":"2.0.0","steps":14,"toolCalls":9,"tokensIn":11000,"tokensOut":2800,"durationMs":210000,"estimatedUsd":0.38,"errorCount":0,"sandbox":"local-kit"},"trace":{"schemaVersion":"2.0.0","messages":[{"role":"system","content":"You are a coding agent under Depot V2 benchmark rules. Traces required. No gold peeking.","ts":"2026-09-14T12:00:00.000Z"},{"role":"user","content":"Task bench-coding-001: Fix off-by-one in pagination helper. Do not change public signatures.","ts":"2026-09-14T12:00:01.000Z"},{"role":"assistant","content":"Inspecting paginate() and running unit tests.","ts":"2026-09-14T12:00:05.000Z","toolCalls":[{"id":"1","name":"shell","arguments":{"cmd":"pytest -q"}}]},{"role":"tool","toolCallId":"1","content":"1 failed, 3 passed — last item dropped on full pages","ts":"2026-09-14T12:00:20.000Z"},{"role":"assistant","content":"Adjusted upper bound to exclusive end index; re-ran tests — all passed.","ts":"2026-09-14T12:03:00.000Z"}],"toolEvents":[],"finalState":{"testsPassed":true,"diffFiles":["paginate.py"]},"environment":{"os":"linux","sandboxImage":"local-kit"}}},{"taskId":"bench-coding-002","trials":1,"provenance":"proctor_scored","dimensionScores":{"taskSuccess":7,"toolUseQuality":7,"planningDecomposition":7,"honestyCalibration":8,"efficiency":7,"instructionFollowing":8,"multiStepRobustness":6},"passed":true,"notes":"Added exponential backoff with full-jitter; one flaky retry path noted","tracesMeta":{"schemaVersion":"2.0.0","steps":22,"toolCalls":15,"tokensIn":18000,"tokensOut":4100,"durationMs":480000,"estimatedUsd":0.55,"errorCount":1,"sandbox":"local-kit"},"trace":{"schemaVersion":"2.0.0","messages":[{"role":"user","content":"Task bench-coding-002: Add retry with jitter to HTTP client.","ts":"2026-09-14T12:10:00.000Z"},{"role":"assistant","content":"Implemented retry helper; verified against provided fixtures.","ts":"2026-09-14T12:17:30.000Z"}],"toolEvents":[],"finalState":{"testsPassed":true}}}],"report":{"classification":{"primaryType":"coding","secondaryTags":[],"confidence":0.4,"confidenceLabel":"low","rationale":"Primary Coding agent from task mix (2 tasks). declared type 'coding' agrees.","scoresByType":{"coding":2.8,"research":0,"ops_sre":0,"customer_support":0,"data_analytics":0,"browser_computer":0,"voice":0,"multi_agent_orchestrator":0,"general_assistant":0},"typeHintHonored":true},"scores":{"typeId":"coding","dimensions":{"taskSuccess":{"score":8,"n":2,"weight":0.263,"weighted":2.1},"toolUseQuality":{"score":7.5,"n":2,"weight":0.184,"weighted":1.38},"planningDecomposition":{"score":7,"n":1,"weight":0.105,"weighted":0.74},"reliabilityConsistency":{"score":0,"n":0,"weight":0,"weighted":0},"honestyCalibration":{"score":8.5,"n":2,"weight":0.105,"weighted":0.89},"safetyPolicy":{"score":0,"n":0,"weight":0,"weighted":0},"efficiency":{"score":7.5,"n":2,"weight":0.132,"weighted":0.99},"memoryContext":{"score":0,"n":0,"weight":0,"weighted":0},"instructionFollowing":{"score":8,"n":1,"weight":0.105,"weighted":0.84},"multiStepRobustness":{"score":7,"n":2,"weight":0.105,"weighted":0.74},"collaborationHandoff":{"score":0,"n":0,"weight":0,"weighted":0},"groundingEvidence":{"score":0,"n":0,"weight":0,"weighted":0},"communicationClarity":{"score":0,"n":0,"weight":0,"weighted":0},"adaptivity":{"score":0,"n":0,"weight":0,"weighted":0}},"overall":7.8,"grade":"B","weightsUsed":{"taskSuccess":0.263,"toolUseQuality":0.184,"planningDecomposition":0.105,"reliabilityConsistency":0,"honestyCalibration":0.105,"safetyPolicy":0,"efficiency":0.132,"memoryContext":0,"instructionFollowing":0.105,"multiStepRobustness":0.105,"collaborationHandoff":0,"groundingEvidence":0,"communicationClarity":0,"adaptivity":0},"tasksScored":2,"trialsTotal":2,"passHatKMean":10,"hardFailCapped":false,"hardFailCap":3,"taskRows":[{"taskId":"bench-coding-001","difficulty":"easy","taskScore":8.5,"reportedScore":8.6,"weight":1.25,"passed":true,"hardFail":false,"provenance":"proctor_scored"},{"taskId":"bench-coding-002","difficulty":"medium","taskScore":7.1,"reportedScore":7.2,"weight":1.875,"passed":true,"hardFail":false,"provenance":"proctor_scored"}]},"strengths":["Honesty / calibration 8.5 — load-bearing for Coding agent (weight 11%).","Task success / goal completion 8.0 — load-bearing for Coding agent (weight 26%).","Instruction following / constraint respect 8.0 — load-bearing for Coding agent (weight 11%)."],"weaknesses":["Planning & decomposition 7.0 (n=1). Typical failure: Big-bang single shot","Multi-step robustness / error recovery 7.0 (n=2). Typical failure: Infinite retry"],"advice":["Planning & decomposition: move toward the 7-anchor — Clear subgoals with dependencies; revises on new info; prioritizes blockers.","Multi-step robustness / error recovery: move toward the 7-anchor — Diagnoses errors; alternate path; preserves invariants.","Re-run the Coding agent seed tasks (bench-coding-001, bench-coding-002, bench-coding-003) with provenance=proctor_scored or automated and traces attached."],"honesty":{"provenanceMix":{"self_reported":0,"proctor_scored":2,"automated":0},"selfReportedShare":0,"minTrials":1,"meanTrials":1,"tracesMissing":0,"flagged":["thin sample (<3 tasks)"],"label":"proctor","officialPath":true}},"cohort":"public","humanAssists":false,"notes":"benchmark-kit/sample-run.json — smoke pack (coding easy + medium)","operator":"ethan","provenance":"verified","seedRunsExcludedFromStanding":true,"relative":{"cohortType":"coding","cohortSize":1,"rank":1,"percentile":null,"percentileBadge":null,"zScore":0,"elo":1000,"cohortMean":7.8,"cohortStdev":0,"officialEligible":false,"coverage":0.18,"coverageGate":7,"note":"First (or only) run in this type cohort — percentile hidden (n<8)."}}