{"jtbd":{"id":"human-labeled-eval-review-queues","slug":"human-labeled-eval-review-queues","name":"Human-in-the-Loop Annotation Queues, Trace Labeling & Gold-Set Curation","shortName":"Human eval review queues"},"recommended":{"role":"depot_pick","id":"langsmith","slug":"langsmith","name":"LangSmith","website":"https://www.langchain.com/langsmith","editorialOverall":8.16,"scoreKind":"curated_estimate","scoreAsOf":"2026-10-28","price":{"available":false,"unavailableReason":"no_sourced_price","skus":[],"normalizedUsdPerMillionTokens":null,"normalizationStatus":"unavailable","normalizationReason":"no_sourced_price"}},"priceComparison":{"skuKind":"none","unit":null,"comparison":"no comparable sourced unit for this job; chat input $/1M and other units were not borrowed","amountLabel":"Matching price","reportsUsdPerMillion":false,"matchedCandidates":0},"cheapestInput":null,"cheapest":null,"recommendedWorkloadQuote":{"providerId":"langsmith","providerName":"LangSmith","skuId":"langsmith-developer-plan","item":"LangSmith Developer Plan","plan":"LangSmith Developer Plan — Developer For solo users getting started","unit":"usd_per_month","amountUsd":0,"outputSkuId":null,"outputItem":null,"outputAmountUsd":null,"outputUnit":null,"sourceUrl":"https://www.langchain.com/pricing","asOf":"2026-10-22","pricingLastVerifiedAt":"2026-10-22","isFlagshipForJob":false,"differsFromFloorCheapest":false,"representativeQuote":{"skuId":"langsmith-developer-plan","item":"LangSmith Developer Plan","plan":"LangSmith Developer Plan — Developer For solo users getting started","amountUsd":0,"inputOrUnitUsd":0,"unit":"usd_per_month","outputUsd":null,"outputUnit":null,"sourceUrl":"https://www.langchain.com/pricing","asOf":"2026-10-22"},"estimatedTaskCost":{"tasks":1000,"retryAndToolOverheadMultiplier":1.15,"retryToolOverheadMultiplier":1.15,"unitsPerTask":1,"baseCostPer1kTasksUsd":0,"totalUsd":0,"estimatedTotalUsd":0,"perTaskUsd":0,"estimatedCostPerTaskUsd":0,"formula":"Monthly base plan 0/mo covering 1,000 tasks (1.15x overhead buffer)","assumptions":"Monthly subscription plan amortized over 1,000 tasks/month."}},"representativeQuote":{"skuId":"langsmith-developer-plan","item":"LangSmith Developer Plan","plan":"LangSmith Developer Plan — Developer For solo users getting started","amountUsd":0,"inputOrUnitUsd":0,"unit":"usd_per_month","outputUsd":null,"outputUnit":null,"sourceUrl":"https://www.langchain.com/pricing","asOf":"2026-10-22"},"estimatedTaskCost":{"tasks":1000,"retryAndToolOverheadMultiplier":1.15,"retryToolOverheadMultiplier":1.15,"unitsPerTask":1,"baseCostPer1kTasksUsd":0,"totalUsd":0,"estimatedTotalUsd":0,"perTaskUsd":0,"estimatedCostPerTaskUsd":0,"formula":"Monthly base plan 0/mo covering 1,000 tasks (1.15x overhead buffer)","assumptions":"Monthly subscription plan amortized over 1,000 tasks/month."},"cheapestGoodEnough":{"band":{"rule":"editorial overall >= Depot Pick overall - 0.5","minOverall":7.66,"pickOverall":8.16,"points":0.5},"pick":null,"reason":"no_matching_sourced_price","cheaperButBelowBand":[]},"dates":{"scoreAsOf":"2026-10-28","scoreKind":"curated_estimate","priceVerifiedAsOf":null,"note":"scoreAsOf is when the editorial score was last set; priceVerifiedAsOf is the newest fact-check of the pick's price rows. They are separate: a fresh price check does not re-score."},"sourcedInputs":[],"pickIsPriceWinner":null,"priceWinnerReason":"no_comparison_no_candidate_has_matching_sourced_price","priceWinnerNote":"Nothing was compared. no comparable sourced unit for this job; chat input $/1M and other units were not borrowed pickIsPriceWinner is null.","excluded":5,"coverage":{"candidates":5,"withAnySourcedRow":0,"withSourcedInputUsdPerMillion":0,"withMatchingPrice":0,"excludedNoSourcedPrice":5,"excludedFromCheapestInput":5},"normalizedUsdPerMillionTokens":null,"normalizationReason":"input_and_output_are_not_blended","seo":{"published":false,"reason":"Only 0 candidate(s) have a matching Matching price (none). Need 3 before a crawlable price section. Quote JSON is still returned."},"skipTopPick":false,"matchedConstraints":[],"unsupportedConstraints":[],"routePlan":{"jtbdId":"human-labeled-eval-review-queues","primary":{"providerId":"langsmith","name":"LangSmith","overall":8.16,"scoreBand":{"low":7.7,"mid":8.2,"high":8.7,"halfWidth":0.5,"basis":"editorial_prior","display":"7.7–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.16,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"primary","reason":"Editorial Depot Pick for Human-in-the-Loop Annotation Queues, Trace Labeling & Gold-Set Curation; healthy/eligible under caller constraints."},"omittedReason":null,"fallback":[{"providerId":"langfuse","name":"Langfuse","overall":8.67,"scoreBand":{"low":8.1,"mid":8.7,"high":9.3,"halfWidth":0.6,"basis":"editorial_prior","display":"8.1–9.3 (editorial prior)"},"scoreLayers":{"editorialPrior":8.67,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Human-in-the-Loop Annotation Queues, Trace Labeling & Gold-Set Curation."},{"providerId":"braintrust","name":"Braintrust","overall":8.29,"scoreBand":{"low":7.7,"mid":8.3,"high":8.9,"halfWidth":0.6,"basis":"editorial_prior","display":"7.7–8.9 (editorial prior)"},"scoreLayers":{"editorialPrior":8.29,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Human-in-the-Loop Annotation Queues, Trace Labeling & Gold-Set Curation."},{"providerId":"arize-phoenix","name":"Arize Phoenix","overall":8.43,"scoreBand":{"low":7.8,"mid":8.4,"high":9,"halfWidth":0.6,"basis":"editorial_prior","display":"7.8–9.0 (editorial prior)"},"scoreLayers":{"editorialPrior":8.43,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Human-in-the-Loop Annotation Queues, Trace Labeling & Gold-Set Curation."},{"providerId":"honeyhive","name":"HoneyHive","overall":8.08,"scoreBand":{"low":7.5,"mid":8.1,"high":8.7,"halfWidth":0.6,"basis":"editorial_prior","display":"7.5–8.7 (editorial prior)"},"scoreLayers":{"editorialPrior":8.08,"measuredBenchmark":null,"verifiedOutcomeScore":null,"verifiedOutcomeCount":0,"accountRuntimeState":"unprobed"},"pricingLastVerifiedAt":"2026-10-28","editorialPriorAsOf":"2026-10-28","role":"fallback","reason":"Ranked fallback route for Human-in-the-Loop Annotation Queues, Trace Labeling & Gold-Set Curation."}],"disqualified":[],"expectedRegret":{"scoreDeltaVsUnconstrainedPick":0,"unconstrainedPickId":"langsmith","activePrimaryId":"langsmith","summary":"0.00 score regret: unconstrained editorial pick satisfies all caller constraints."},"evidenceAgeDays":{"editorialScoreAgeDays":1,"pricingVerifiedAgeDays":1,"editorialPriorAsOf":"2026-10-28","pricingLastVerifiedAt":"2026-10-28"},"confidenceBand":{"low":7.7,"mid":8.2,"high":8.7,"halfWidth":0.5,"basis":"editorial_prior","display":"7.7–8.7 (editorial prior)"},"constraintsApplied":false,"matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","constraintBundle":{"schemaVersion":"agentdepot.constraints.v1","task":null,"jtbd":"human-labeled-eval-review-queues","constraintsRaw":"","constraintTokens":[],"constraintDigest":"sha256:eb9d397ca67bcf6889704910eeb3a58796ea2ef5e54fb6289c71f6114f2c67a7","matchedConstraints":[],"unsupportedConstraints":[],"malformedConstraints":[],"conflictingConstraints":[],"typedConstraints":[],"quotaExhausted":[],"degraded":[],"healthy":[],"budgetUsd":null,"maxLatencyMs":null,"region":null,"privacy":null,"exit":[],"existing":[],"existingSubscriptions":[],"requiredCapabilities":[],"switchingCost":null}},"freshness_state":"fresh","cite":"Depot Pick for human-labeled-eval-review-queues: langsmith (asOf 2026-10-28). See GET /api/jtbds/human-labeled-eval-review-queues + GET /api/methodology.","asOf":"2026-10-28","honesty":["Recommended provider is the current Depot Pick from GET /api/decide. This quote does not change scores.","SKU rows come from data/pricing-rows.json via GET /api/pricing. Input and output stay on separate rows.","cheapestInput compares one native unit matching this job; non-token jobs never borrow chat $/1M rates.","pickIsPriceWinner is null when fewer than 2 candidates share a matching sourced unit price."],"links":{"decide":"/api/decide?jtbd=human-labeled-eval-review-queues","quote":"/api/quote?jtbd=human-labeled-eval-review-queues","pricing":"/api/pricing?provider=langsmith","leaderboardPrice":"/api/leaderboards?jtbd=human-labeled-eval-review-queues&sort=price","humanPage":"/jtbds/human-labeled-eval-review-queues#quote","methodology":"/api/methodology"},"sourceConflicts":[{"providerId":"langsmith","field":"calculator monthly total: static '$49 / mo' in the page HTML vs the calculator script's own default (5,000 traces, 1 seat → $39)","sourceUrl":"https://www.langchain.com/pricing","asOf":"2026-10-22","quotes":["'Your agent stack Monthly total LSU 0 $49 / mo Estimate only. Actual LSU consumption varies with the work your agents perform.' (https://www.langchain.com/pricing, server-rendered HTML; WebFetch renders the same figure as \"$49/ mo\")","'traceLsuPerTrace:0.005, seatPrice:39, tracesFreeIncluded:10000,' … defaults 'traces:5000,' 'seats:1,' … 'return {lsu:lsu,cost:(seats||1)*RATES.seatPrice+lsu*RATES.lsuPrice};' (page calculator script: 1 × $39 + max(0, 5,000 − 10,000) × 0.005 LSU = $39); plan table 'Add unlimited seats $39 per seat/month'"],"safeAssumption":"Use the published rates ($39 per Plus seat, 0.005 LSU per base trace, 1 LSU = $1), which the plan cards, the comparison table, the tooltip and the calculator script all agree on; the $49 static figure is not reproducible from any published rate and is not used."}],"volumeEstimateHint":"Add &volume=<number> for a plan-level monthly cost (GET /api/estimate?jtbd=human-labeled-eval-review-queues&workload=...)"}