{"id":"human-labeled-eval-review-queues","slug":"human-labeled-eval-review-queues","name":"Human-in-the-Loop Annotation Queues, Trace Labeling & Gold-Set Curation","shortName":"Human eval review queues","problemStatement":"Route low-confidence production traces, thumbs-down user sessions, and edge-case failures into structured human review queues so domain experts can label failure modes and promote curated examples into your golden eval dataset.","whenNotTopPick":"Open-source / self-hosted annotation queues and dataset curation — Langfuse; eval-first human review + scorer alignment — Braintrust; OpenTelemetry OSS trace labeling — Arize Phoenix; rapid team trace curation — HoneyHive.","categoryIds":["cat-observability"],"depotPickProviderId":"langsmith","depotPickRationale":"LangSmith's native Annotation Queues let reviewers step through multi-turn agent traces with rubric criteria, attach structured feedback scores, and one-click export corrected trajectories into versioned evaluation datasets (Developer $0 up to 5k base traces, Plus $39/seat as of 2026-10-22).","runnersUpIds":["langfuse","braintrust","arize-phoenix","honeyhive"],"weights":{"capabilityFit":0.26,"pricePerformance":0.16,"latencyReliability":0.16,"developerDx":0.16,"privacyPosture":0.12,"lockInRisk":0.06,"maturity":0.08},"methodologyNotes":"Synthetic data never fully replaces domain-expert review on real production failures. Sample 1–5% of production traces (weighted toward tool errors, high retry counts, user edits, and near-tie judge scores) into a weekly human review queue to keep your golden dataset grounded and calibrate LLM judges.","tags":["evals","annotation","human-in-the-loop","hitl","datasets","review-queue","curation"],"scoreMargin":-0.51,"isNearTie":false,"pickConfidenceBand":"moderate_lead","nearTie":{"isNearTie":false,"scoreMargin":-0.51,"threshold":0.35,"pickConfidenceBand":"moderate_lead","coPickProviderId":"langfuse","coPickProviderName":"Langfuse","coPickScore":8.67,"decidingFactor":"latencyReliability (8.8 vs 8.4)","methodologyWarning":null,"tieBreakerRule":"Open-source / self-hosted annotation queues and dataset curation — Langfuse; eval-first human review + scorer alignment — Braintrust; OpenTelemetry OSS trace labeling — Arize Phoenix; rapid team trace curation — HoneyHive."},"evidenceStatus":{"decisionRole":"independent_prior_and_candidate_generator","positioning":"The independent, agent-readable prior for choosing AI infrastructure — not a universal router, generic marketplace, or synthetic leaderboard.","freshnessThresholdDays":30,"freshness_policy":{"policy_version":"agentdepot.freshness_policy.v1","threshold_days":30},"freshness_state":"fresh","editorial_freshness_state":"fresh","unqualified_current_pick_allowed":true,"pick_qualification":"qualified_current_pick","promotionClass":"deterministic_source_reverified","evidenceClass":"editorial_prior","staleDowngraded":false,"effectiveConfidence":"high","confidenceHalfWidth":0.5,"pickScoreAgeDays":0,"editorialJudgmentAgeDays":0,"pricingAgeDays":0,"isPickScoreStale":false,"isPricingStale":false,"editorialJudgmentAsOf":"2026-10-28","pricingVerifiedAt":"2026-10-28","outcomeVerifiedAt":null,"effectiveVerifiedAsOf":"2026-10-28","downgradeReasons":[],"stalenessWarning":"Editorial judgment and deterministic pricing verification are both within the 30-day freshness threshold.","promotionGate":{"deterministicPricingCheck":true,"approvedScoreReaffirm":true,"pendingHumanReviewBlock":false,"zeroUnverifiedSkus":true,"costEfficiencyConsistent":true,"lockInAndReliabilityConsistent":true,"secondModelOrRuleAudit":true,"countersignReceiptIds":[],"otsAttestationHead":"b32634d5784cbfe663be0a16cce4f60a3ff1c3d4490437be00361cb4ae5d5d59","sourceCheckDate":"2026-10-28","activePricingRowsCount":0,"freeResourcesCount":0,"runnerUpPricingRowsCount":0},"claimProvenance":{"claim_id":"claim:human-labeled-eval-review-queues:langsmith:depot-pick","provider_id":"langsmith","jtbd_id":"human-labeled-eval-review-queues","primary_url":"https://www.langchain.com/langsmith","source_type":"primary_benchmark_or_status","source_revision":"sha256:968a3a72ca09cb92","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.langchain.com/langsmith","final_url":"https://www.langchain.com/langsmith","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_benchmark_or_status","is_claim_specific":true,"source_revision":"sha256:968a3a72ca09cb92","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"verifiedOutcomeSummary":{"totalVerifiedReceipts":0,"byEvidenceTier":{"E0_ASSERTION":0,"E1_SOURCE_BOUND":0,"E2_DELIVERY":0,"E3_INDEPENDENT_OUTCOME":0,"E4_REPLICATED_OUTCOME":0},"bySubjectLayer":{"provider":0,"product":0,"model_version":0,"runtime_route":0}}},"pickScoreDetails":{"scoreRange":[7.66,8.66],"formattedScore":"8.2 ± 0.5 (7.7–8.7 · editorial_prior)","evidenceClass":"editorial_prior","promotionClass":"deterministic_source_reverified","staleDowngraded":false,"freshState":"fresh","freshness_state":"fresh","effectiveConfidence":"high","confidenceInterval":{"low":7.66,"mid":8.16,"high":8.66,"halfWidth":0.5,"confidenceLevel":0.8,"basis":"deterministic_source_reverified_interval_0.5"},"editorialJudgmentAsOf":"2026-10-28","sourceAgeDays":0,"pricingVerifiedAt":"2026-10-28","pricingAgeDays":0,"outcomeVerifiedAt":null,"effectiveVerifiedAsOf":"2026-10-28","dissentingEvidence":["LangChain-centric gravity (works elsewhere but shines in-family)","Cost scales with trace volume","Lock-in to LangSmith workflows","When not to use top pick: Open-source / self-hosted annotation queues and dataset curation — Langfuse; eval-first human review + scorer alignment — Braintrust; OpenTelemetry OSS trace labeling — Arize Phoenix; rapid team trace curation — HoneyHive.","Runner-up Langfuse (langfuse) is the primary alternative when constraints disqualify LangSmith."],"whatWouldChangePick":["Caller constraint matching: Open-source / self-hosted annotation queues and dataset curation — Langfuse; eval-first human review + scorer alignment — Braintrust; OpenTelemetry OSS trace labeling — Arize Phoenix; rapid team trace curation — HoneyHive.","Live quota or rate-limit exhaustion on langsmith (pass ?exhausted=langsmith or liveQuota.exhaustedProviders=[\"langsmith\"] to promote langfuse).","Active subscription or enterprise commit on langfuse combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.langchain.com/langsmith or unverified SKU drift detected by npm run check:sources."],"claimProvenance":{"claim_id":"claim:human-labeled-eval-review-queues:langsmith:depot-pick","provider_id":"langsmith","jtbd_id":"human-labeled-eval-review-queues","primary_url":"https://www.langchain.com/langsmith","source_type":"primary_benchmark_or_status","source_revision":"sha256:968a3a72ca09cb92","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.langchain.com/langsmith","final_url":"https://www.langchain.com/langsmith","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_benchmark_or_status","is_claim_specific":true,"source_revision":"sha256:968a3a72ca09cb92","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"calibratedScore":{"point":8.16,"band":[7.66,8.66],"interval":{"lower":7.66,"upper":8.66,"half_width":0.5},"score_kind":"source_bound_prior_band","evidence_class":"source_bound","confidence":"medium","confidence_ceiling":"medium","age_days":0,"dissent":["LangChain-centric gravity (works elsewhere but shines in-family)","Cost scales with trace volume","Lock-in to LangSmith workflows","When not to use top pick: Open-source / self-hosted annotation queues and dataset curation — Langfuse; eval-first human review + scorer alignment — Braintrust; OpenTelemetry OSS trace labeling — Arize Phoenix; rapid team trace curation — HoneyHive.","Runner-up Langfuse (langfuse) is the primary alternative when constraints disqualify LangSmith."],"what_would_change_this":["Caller constraint matching: Open-source / self-hosted annotation queues and dataset curation — Langfuse; eval-first human review + scorer alignment — Braintrust; OpenTelemetry OSS trace labeling — Arize Phoenix; rapid team trace curation — HoneyHive.","Live quota or rate-limit exhaustion on langsmith (pass ?exhausted=langsmith or liveQuota.exhaustedProviders=[\"langsmith\"] to promote langfuse).","Active subscription or enterprise commit on langfuse combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.langchain.com/langsmith or unverified SKU drift detected by npm run check:sources."],"subject_layer":"provider"},"subject_ref":{"layer":"provider","provider_id":"langsmith","product_id":"langsmith-core","model_id":null,"version":null,"route_class":null,"inheritance_policy":"no_cross_layer_inheritance"}},"runtimeRoutes":[{"routeId":"human-labeled-eval-review-queues:langsmith:primary","roleForJob":"depot_pick","provider":{"id":"langsmith","slug":"langsmith","name":"LangSmith","website":"https://www.langchain.com/langsmith","status":"active"},"model":{"representativeModelId":"LangSmith Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.langchain.com/langsmith","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.langchain.com/langsmith (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"LangSmith Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"Developer: Developer For solo users getting started. $0 / seat per month then pay as you go Start for","subscriptionFirstEligible":true,"documentedPlansCount":2},"regions":{"supportedRegions":["us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"interactive_sub_1s","latencyReliabilityScore":8.8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":7.8,"zeroDataRetentionAvailable":false,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"langfuse"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:langsmith:langsmith-primary:managed_cloud_service:us","provider_id":"langsmith","product_id":"langsmith-core","model_id":"langsmith-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:langsmith:langsmith-primary:direct_api:us","provider_id":"langsmith","product_id":"langsmith-core","model_id":"langsmith-primary","version":"2026-10-28","route_class":"direct_api","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"human-labeled-eval-review-queues:langfuse:primary","roleForJob":"runner_up","provider":{"id":"langfuse","slug":"langfuse","name":"Langfuse","website":"https://langfuse.com","status":"active"},"model":{"representativeModelId":"Langfuse Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://langfuse.com","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://langfuse.com (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Langfuse Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":true,"freeTierSummary":"Langfuse self-host (oss, asOf 2026-10-25)","rateLimitsSummary":"Hobby: Hobby (Free) … 50k units / month included · 30 days data access · 2 users … Included usage","subscriptionFirstEligible":true,"documentedPlansCount":4},"regions":{"supportedRegions":["eu","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.4,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"langsmith"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:langfuse:langfuse-primary:self_hosted_or_oss_compatible:eu","provider_id":"langfuse","product_id":"langfuse-core","model_id":"langfuse-primary","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"eu","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:langfuse:langfuse-primary:direct_api:eu","provider_id":"langfuse","product_id":"langfuse-core","model_id":"langfuse-primary","version":"2026-10-28","route_class":"direct_api","region":"eu","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"human-labeled-eval-review-queues:braintrust:primary","roleForJob":"runner_up","provider":{"id":"braintrust","slug":"braintrust","name":"Braintrust","website":"https://www.braintrust.dev","status":"active"},"model":{"representativeModelId":"Braintrust Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.braintrust.dev","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.braintrust.dev (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Braintrust Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"Starter: Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - ","subscriptionFirstEligible":true,"documentedPlansCount":2},"regions":{"supportedRegions":["us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.6,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"langsmith"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:managed_cloud_service:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:direct_api:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"direct_api","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"human-labeled-eval-review-queues:arize-phoenix:primary","roleForJob":"runner_up","provider":{"id":"arize-phoenix","slug":"arize-phoenix","name":"Arize Phoenix","website":"https://phoenix.arize.com","status":"active"},"model":{"representativeModelId":"Arize Phoenix Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://phoenix.arize.com","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://phoenix.arize.com (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Arize Phoenix Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=arize-phoenix","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":false,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.2,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.5,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"langsmith"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:arize-phoenix:arize-phoenix-primary:self_hosted_or_oss_compatible:global","provider_id":"arize-phoenix","product_id":"arize-phoenix-core","model_id":"arize-phoenix-primary","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"global","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:arize-phoenix:arize-phoenix-primary:direct_api:global","provider_id":"arize-phoenix","product_id":"arize-phoenix-core","model_id":"arize-phoenix-primary","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]}],"depth":{"whenNotToUse":{"text":"Open-source / self-hosted annotation queues and dataset curation — Langfuse; eval-first human review + scorer alignment — Braintrust; OpenTelemetry OSS trace labeling — Arize Phoenix; rapid team trace curation — HoneyHive.","source":"editorial `whenNotTopPick` (data/jtbds.json)"},"whenNotToUseReason":null,"runnerUp":{"providerId":"langfuse","name":"Langfuse","overallForJob":8.67,"pickOverallForJob":8.16,"beatsPickOn":["pricePerformance","privacyPosture","lockInRisk"],"source":"first resolvable entry of the job's editorially ranked `runnersUpIds`; scores are stored job-weighted scores"},"runnerUpReason":null,"lockIn":{"pickLockInRisk":6.5,"runnerUpLockInRisk":2.5,"scale":"0–10 stored curated estimate; higher = more lock-in (worse)","pickExitSignals":[],"runnerUpExitSignals":["open source (provider tag `oss`)"],"pickNotes":[{"text":"Lock-in to LangSmith workflows","field":"weaknesses"}],"selfHostFallback":[],"migration":"Runner-up Langfuse has lower stored lock-in risk (2.5 vs 6.5, 0–10, higher = more lock-in).","migrationReason":null,"derivedFrom":"stored lockInRisk scores, provider tags (oss, self-host, openai-compat, open-models), free rows that list this job, and verbatim provider notes. No migration guide was invented."}},"freshness":{"pick":{"score":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"lastReviewed":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"sourceCheck":{"checkedAt":"2026-10-28","sourceUrls":1,"sourcesReachable":1,"pricingRows":0,"pricingRowsReconfirmed":0,"note":"A source re-check confirms the page is reachable and, for pricing rows, that the stored amount and SKU name appear on it. It does not re-derive or change the score.","reason":null}}},"whenNotTopPickRules":[{"ifConstraint":"privacy:zdr_or_compliance_strict","recommendedProviderId":"langfuse","activeModelOrSku":"langfuse-cloud-core","reason":"Open-source / self-hosted annotation queues and dataset curation — Langfuse"},{"ifConstraint":"escape_hatch_clause_2","recommendedProviderId":"braintrust","activeModelOrSku":"braintrust-pro","reason":"eval-first human review + scorer alignment — Braintrust"},{"ifConstraint":"exit:self_host_or_open_weights","recommendedProviderId":"arize-phoenix","activeModelOrSku":"arize-phoenix-standard","reason":"OpenTelemetry OSS trace labeling — Arize Phoenix"},{"ifConstraint":"escape_hatch_clause_4","recommendedProviderId":"honeyhive","activeModelOrSku":"honeyhive-standard","reason":"rapid team trace curation — HoneyHive."}],"bootstrap":{"authType":"api_key","envVars":["LANGSMITH_API_KEY","LANGSMITH_TRACING"],"signupUrl":"https://smith.langchain.com/","apiKeyUrl":"https://smith.langchain.com/settings","docsUrl":"https://docs.smith.langchain.com/","mcpServer":null,"cliInit":null,"sdkInstall":{"npm":"npm install langsmith","pip":"pip install langsmith"},"installCommand":"npm install langsmith","activeModelOrSku":"langsmith-plus","canonicalSnippet":"import { traceable } from \"langsmith/traceable\";\nconst runAgent = traceable(async (input: string) => input, { name: \"agent_step\" });","verifyCommand":"curl -sS https://api.smith.langchain.com/api/v1/info -H \"x-api-key: $LANGSMITH_API_KEY\""},"quickstartSpec":{"installCommand":"npm install langsmith","envVars":["LANGSMITH_API_KEY","LANGSMITH_TRACING"],"activeModelOrSku":"langsmith-plus","canonicalSnippet":"import { traceable } from \"langsmith/traceable\";\nconst runAgent = traceable(async (input: string) => input, { name: \"agent_step\" });","sdkInstall":{"npm":"npm install langsmith","pip":"pip install langsmith"},"verifyCommand":"curl -sS https://api.smith.langchain.com/api/v1/info -H \"x-api-key: $LANGSMITH_API_KEY\"","apiKeyUrl":"https://smith.langchain.com/settings","docsUrl":"https://docs.smith.langchain.com/","mcpServer":null,"bootstrapReason":null},"operationalGotchas":[{"code":"hidden_spend_tier_gates","severity":"warning","summary":"LangChain-centric gravity (works elsewhere but shines in-family)","sourceDate":"2026-10-28"},{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Cost scales with trace volume","sourceDate":"2026-10-28"}],"relatedPacks":[],"depotPick":{"id":"langsmith","slug":"langsmith","name":"LangSmith","tagline":"If you live in LangGraph, LangSmith is the obvious observatory. Otherwise compare Langfuse/Helicone first.","overall":8.16},"runnersUp":[{"id":"langfuse","slug":"langfuse","name":"Langfuse","tagline":"Depot default observability for most agent teams: open, affordable, good enough to serious.","overall":8.67},{"id":"braintrust","slug":"braintrust","name":"Braintrust","tagline":"Depot pick when evals are the product quality gate. Pair with Helicone/Langfuse for cheap request logs if…","overall":8.29},{"id":"arize-phoenix","slug":"arize-phoenix","name":"Arize Phoenix","tagline":"Choose Phoenix when you want open-source observability you can own. Cloud Arize when enterprise wants a…","overall":8.43},{"id":"honeyhive","slug":"honeyhive","name":"HoneyHive","tagline":"Use the free Developer tier to stand up agent evals/traces in a day; Enterprise when you need self-host…","overall":8.08}]}