{"id":"agent-trajectory-tool-use-evals","slug":"agent-trajectory-tool-use-evals","name":"Agent Trajectory, Tool-Call & Multi-Step Plan Evaluation","shortName":"Agent trajectory & tool-use evals","problemStatement":"Grade an agent's multi-step execution trajectory — tool selection, JSON argument schemas, call ordering, state mutations, and retry recovery — instead of only scoring the final text string.","whenNotTopPick":"LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo.","categoryIds":["cat-observability"],"depotPickProviderId":"braintrust","depotPickRationale":"Braintrust links nested span trees (tool calls, arguments, intermediate states) directly to dataset scorers and CI diff views, so you can grade trajectory efficiency, redundant tool loops, and final outcome together.","runnersUpIds":["langsmith","arize-phoenix","langfuse","promptfoo"],"weights":{"capabilityFit":0.28,"pricePerformance":0.14,"latencyReliability":0.14,"developerDx":0.18,"privacyPosture":0.1,"lockInRisk":0.06,"maturity":0.1},"methodologyNotes":"Weights capabilityFit and developerDx highest: trajectory evals must inspect intermediate tool spans, argument validity, and unnecessary retries alongside final task completion. Pair with BFCL (https://gorilla.cs.berkeley.edu/leaderboard.html) for function-calling schema accuracy and τ-bench (https://github.com/sierra-research/tau-bench) for policy-compliant tool sequences.","tags":["evals","trajectory","tool-use","function-calling","traces","benchmarking"],"scoreMargin":0.11,"isNearTie":true,"pickConfidenceBand":"near_tie_co_pick","nearTie":{"isNearTie":true,"scoreMargin":0.11,"threshold":0.35,"pickConfidenceBand":"near_tie_co_pick","coPickProviderId":"langsmith","coPickProviderName":"LangSmith","coPickScore":8.22,"decidingFactor":"pricePerformance (7.5 vs 7)","methodologyWarning":"Top pick (8.33) and #2 runner-up (8.22) are separated by 0.11 points (<= 0.35 weight-sensitivity band). Treat them as co-picks and use the workload tie-breaker rule below rather than the raw composite decimal.","tieBreakerRule":"LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo."},"evidenceStatus":{"decisionRole":"independent_prior_and_candidate_generator","positioning":"The independent, agent-readable prior for choosing AI infrastructure — not a universal router, generic marketplace, or synthetic leaderboard.","freshnessThresholdDays":30,"freshness_policy":{"policy_version":"agentdepot.freshness_policy.v1","threshold_days":30},"freshness_state":"fresh","editorial_freshness_state":"fresh","unqualified_current_pick_allowed":true,"pick_qualification":"qualified_current_pick","promotionClass":"verified_outcome_backed","evidenceClass":"editorial_plus_verified_receipt","staleDowngraded":false,"effectiveConfidence":"high_verified","confidenceHalfWidth":0.35,"pickScoreAgeDays":0,"editorialJudgmentAgeDays":9,"pricingAgeDays":null,"isPickScoreStale":false,"isPricingStale":true,"editorialJudgmentAsOf":"2026-10-19","pricingVerifiedAt":null,"outcomeVerifiedAt":"2026-10-28","effectiveVerifiedAsOf":"2026-10-28","downgradeReasons":[],"stalenessWarning":"Editorial judgment and deterministic pricing verification are both within the 30-day freshness threshold.","promotionGate":{"deterministicPricingCheck":false,"approvedScoreReaffirm":true,"pendingHumanReviewBlock":false,"zeroUnverifiedSkus":true,"costEfficiencyConsistent":true,"lockInAndReliabilityConsistent":true,"secondModelOrRuleAudit":true,"countersignReceiptIds":["cs-54379c97","cs-6eab5b5d","cs-316f2587","cs-0e63df18","cs-1035466f"],"otsAttestationHead":"b32634d5784cbfe663be0a16cce4f60a3ff1c3d4490437be00361cb4ae5d5d59","sourceCheckDate":"2026-10-27","activePricingRowsCount":0,"freeResourcesCount":0,"runnerUpPricingRowsCount":1},"claimProvenance":{"claim_id":"claim:agent-trajectory-tool-use-evals:braintrust:depot-pick","provider_id":"braintrust","jtbd_id":"agent-trajectory-tool-use-evals","primary_url":"https://www.braintrust.dev/docs","source_type":"primary_docs","source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.braintrust.dev","final_url":"https://www.braintrust.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:fdb47a7e8e944079","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/docs","final_url":"https://www.braintrust.dev/docs","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_docs","is_claim_specific":true,"source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/pricing","final_url":"https://www.braintrust.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:60bb11c5a2c023e3","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"verifiedOutcomeSummary":{"totalVerifiedReceipts":0,"byEvidenceTier":{"E0_ASSERTION":0,"E1_SOURCE_BOUND":0,"E2_DELIVERY":0,"E3_INDEPENDENT_OUTCOME":0,"E4_REPLICATED_OUTCOME":0},"bySubjectLayer":{"provider":0,"product":0,"model_version":0,"runtime_route":0}}},"pickScoreDetails":{"scoreRange":[7.98,8.68],"formattedScore":"8.3 ± 0.35 (8.0–8.7 · editorial_plus_verified_receipt)","evidenceClass":"editorial_plus_verified_receipt","promotionClass":"verified_outcome_backed","staleDowngraded":false,"freshState":"fresh","freshness_state":"fresh","effectiveConfidence":"high_verified","confidenceInterval":{"low":7.98,"mid":8.33,"high":8.68,"halfWidth":0.35,"confidenceLevel":0.8,"basis":"narrowed_verified_outcome_interval_0.35_backed_by_countersign_receipts"},"editorialJudgmentAsOf":"2026-10-28","sourceAgeDays":9,"pricingVerifiedAt":null,"pricingAgeDays":null,"outcomeVerifiedAt":"2026-10-28","effectiveVerifiedAsOf":"2026-10-28","dissentingEvidence":["Broader tracing competitors exist","Price entry higher than hobby tools","Learning curve for eval design","When not to use top pick: LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo.","Runner-up LangSmith (langsmith) is the primary alternative when constraints disqualify Braintrust."],"whatWouldChangePick":["Caller constraint matching: LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo.","Live quota or rate-limit exhaustion on braintrust (pass ?exhausted=braintrust or liveQuota.exhaustedProviders=[\"braintrust\"] to promote langsmith).","Active subscription or enterprise commit on langsmith combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.braintrust.dev or unverified SKU drift detected by npm run check:sources."],"claimProvenance":{"claim_id":"claim:agent-trajectory-tool-use-evals:braintrust:depot-pick","provider_id":"braintrust","jtbd_id":"agent-trajectory-tool-use-evals","primary_url":"https://www.braintrust.dev/docs","source_type":"primary_docs","source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28","claim_support_state":"claim_specific_supported","max_allowed_evidence_class":"replicated_outcome","defect_codes":[],"sources":[{"original_url":"https://www.braintrust.dev","final_url":"https://www.braintrust.dev","redirected":false,"http_status":200,"reachable":true,"source_type":"generic_homepage","is_claim_specific":false,"source_revision":"sha256:fdb47a7e8e944079","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/docs","final_url":"https://www.braintrust.dev/docs","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_docs","is_claim_specific":true,"source_revision":"sha256:2c8a9a8b9cd0b619","observed_at":"2026-10-28","checked_at":"2026-10-28"},{"original_url":"https://www.braintrust.dev/pricing","final_url":"https://www.braintrust.dev/pricing","redirected":false,"http_status":200,"reachable":true,"source_type":"primary_pricing","is_claim_specific":true,"source_revision":"sha256:60bb11c5a2c023e3","observed_at":"2026-10-28","checked_at":"2026-10-28"}]},"calibratedScore":{"point":8.33,"band":[7.98,8.68],"interval":{"lower":7.98,"upper":8.68,"half_width":0.35},"score_kind":"experimental_calibrated_band","evidence_class":"experimental","confidence":"low","confidence_ceiling":"high","age_days":0,"dissent":["Broader tracing competitors exist","Price entry higher than hobby tools","Learning curve for eval design","When not to use top pick: LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo.","Runner-up LangSmith (langsmith) is the primary alternative when constraints disqualify Braintrust."],"what_would_change_this":["Caller constraint matching: LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo.","Live quota or rate-limit exhaustion on braintrust (pass ?exhausted=braintrust or liveQuota.exhaustedProviders=[\"braintrust\"] to promote langsmith).","Active subscription or enterprise commit on langsmith combined with low switching-cost tolerance (existingSubscriptions + switchingCost.maxMigrationEffort=\"low\").","Sourced price increase >20% on https://www.braintrust.dev or unverified SKU drift detected by npm run check:sources."],"subject_layer":"provider"},"subject_ref":{"layer":"provider","provider_id":"braintrust","product_id":"braintrust-core","model_id":null,"version":null,"route_class":null,"inheritance_policy":"no_cross_layer_inheritance"}},"runtimeRoutes":[{"routeId":"agent-trajectory-tool-use-evals:braintrust:primary","roleForJob":"depot_pick","provider":{"id":"braintrust","slug":"braintrust","name":"Braintrust","website":"https://www.braintrust.dev","status":"active"},"model":{"representativeModelId":"Braintrust Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.braintrust.dev","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.braintrust.dev (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Braintrust Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"Starter: Starter For everyone $0 / month - $10 credits + tok rates - 1 GB processed data + $4/GB - ","subscriptionFirstEligible":true,"documentedPlansCount":2},"regions":{"supportedRegions":["us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.6,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"langsmith"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:managed_cloud_service:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:braintrust:braintrust-primary:direct_api:us","provider_id":"braintrust","product_id":"braintrust-core","model_id":"braintrust-primary","version":"2026-10-28","route_class":"direct_api","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"agent-trajectory-tool-use-evals:langsmith:primary","roleForJob":"runner_up","provider":{"id":"langsmith","slug":"langsmith","name":"LangSmith","website":"https://www.langchain.com/langsmith","status":"active"},"model":{"representativeModelId":"LangSmith Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://www.langchain.com/langsmith","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"managed_cloud_service","endpointPattern":"https://www.langchain.com/langsmith (managed cloud service)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"LangSmith Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"Developer: Developer For solo users getting started. $0 / seat per month then pay as you go Start for","subscriptionFirstEligible":true,"documentedPlansCount":2},"regions":{"supportedRegions":["us","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"interactive_sub_1s","latencyReliabilityScore":8.8,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":7.8,"zeroDataRetentionAvailable":false,"trainsOnApiDataByDefault":false,"hipaaEligible":false,"selfHostable":false},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"braintrust"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:langsmith:langsmith-primary:managed_cloud_service:us","provider_id":"langsmith","product_id":"langsmith-core","model_id":"langsmith-primary","version":"2026-10-28","route_class":"managed_cloud_service","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:langsmith:langsmith-primary:direct_api:us","provider_id":"langsmith","product_id":"langsmith-core","model_id":"langsmith-primary","version":"2026-10-28","route_class":"direct_api","region":"us","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"agent-trajectory-tool-use-evals:arize-phoenix:primary","roleForJob":"runner_up","provider":{"id":"arize-phoenix","slug":"arize-phoenix","name":"Arize Phoenix","website":"https://phoenix.arize.com","status":"active"},"model":{"representativeModelId":"Arize Phoenix Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://phoenix.arize.com","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://phoenix.arize.com (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Arize Phoenix Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":false,"freeTierSummary":null,"rateLimitsSummary":"See /api/limits?provider=arize-phoenix","subscriptionFirstEligible":true,"documentedPlansCount":0},"regions":{"supportedRegions":["global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":false,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.2,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.5,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"braintrust"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:arize-phoenix:arize-phoenix-primary:self_hosted_or_oss_compatible:global","provider_id":"arize-phoenix","product_id":"arize-phoenix-core","model_id":"arize-phoenix-primary","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"global","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:arize-phoenix:arize-phoenix-primary:direct_api:global","provider_id":"arize-phoenix","product_id":"arize-phoenix-core","model_id":"arize-phoenix-primary","version":"2026-10-28","route_class":"direct_api","region":"global","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]},{"routeId":"agent-trajectory-tool-use-evals:langfuse:primary","roleForJob":"runner_up","provider":{"id":"langfuse","slug":"langfuse","name":"Langfuse","website":"https://langfuse.com","status":"active"},"model":{"representativeModelId":"Langfuse Primary Runtime","skuId":null,"unit":null,"amountUsd":null,"sourceUrl":"https://langfuse.com","asOf":"2026-10-28"},"servingRoute":{"primaryRouteType":"self_hosted_or_oss_compatible","endpointPattern":"https://langfuse.com (self hosted or oss compatible)","aggregatorAlternatives":[{"aggregatorId":"direct-only","name":"Langfuse Direct API","routeType":"first_party","tradeoff":"Direct vendor SDK/REST contract; pair with runner-up for multi-vendor failover."}]},"accountAndQuota":{"freeTierAvailable":true,"freeTierSummary":"Langfuse self-host (oss, asOf 2026-10-25)","rateLimitsSummary":"Hobby: Hobby (Free) … 50k units / month included · 30 days data access · 2 users … Included usage","subscriptionFirstEligible":true,"documentedPlansCount":4},"regions":{"supportedRegions":["eu","global"],"multiRegionFailover":true,"euResidencyAvailable":true},"toolSchemaSupport":{"strictJsonSchema":true,"parallelToolCalls":false,"mcpServerAvailable":true,"streamingSupported":true},"latencyTier":{"tier":"standard_1s_to_5s","latencyReliabilityScore":8.4,"coldStartNotes":"Standard warm pool; check provider SLA for burst concurrency"},"privacyConfig":{"privacyPostureScore":8.8,"zeroDataRetentionAvailable":true,"trainsOnApiDataByDefault":false,"hipaaEligible":true,"selfHostable":true},"availabilityStatus":{"providerStatus":"active","verifiedActiveSkus":0,"unverifiedSkus":0,"liveFailoverTargetId":"braintrust"},"subject_ref":{"layer":"runtime_route","subject_id":"runtime_route:langfuse:langfuse-primary:self_hosted_or_oss_compatible:eu","provider_id":"langfuse","product_id":"langfuse-core","model_id":"langfuse-primary","version":"2026-10-28","route_class":"self_hosted_or_oss_compatible","region":"eu","score_status":"unknown","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"},"distinctRouteSubjects":[{"layer":"runtime_route","subject_id":"runtime_route:langfuse:langfuse-primary:direct_api:eu","provider_id":"langfuse","product_id":"langfuse-core","model_id":"langfuse-primary","version":"2026-10-28","route_class":"direct_api","region":"eu","score_status":"layer_evidenced","layer_score":"unknown","inherited_from_parent_layer":false,"inheritance_policy":"no_cross_layer_inheritance"}]}],"depth":{"whenNotToUse":{"text":"LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith; self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix; open-source MIT tracing + dataset runs — Langfuse; local CLI tool-call assertions in CI without SaaS — Promptfoo.","source":"editorial `whenNotTopPick` (data/jtbds.json)"},"whenNotToUseReason":null,"runnerUp":{"providerId":"langsmith","name":"LangSmith","overallForJob":8.22,"pickOverallForJob":8.33,"beatsPickOn":["latencyReliability","developerDx","maturity"],"source":"first resolvable entry of the job's editorially ranked `runnersUpIds`; scores are stored job-weighted scores"},"runnerUpReason":null,"lockIn":{"pickLockInRisk":5,"runnerUpLockInRisk":6.5,"scale":"0–10 stored curated estimate; higher = more lock-in (worse)","pickExitSignals":[],"runnerUpExitSignals":[],"pickNotes":[],"selfHostFallback":[],"migration":"The pick has lower stored lock-in risk than runner-up LangSmith (5 vs 6.5).","migrationReason":null,"derivedFrom":"stored lockInRisk scores, provider tags (oss, self-host, openai-compat, open-models), free rows that list this job, and verbatim provider notes. No migration guide was invented."}},"freshness":{"pick":{"score":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"lastReviewed":{"asOf":"2026-10-28","ageDays":1,"stale":false,"staleAfterDays":30,"reason":null},"sourceCheck":{"checkedAt":"2026-10-28","sourceUrls":1,"sourcesReachable":1,"pricingRows":0,"pricingRowsReconfirmed":0,"note":"A source re-check confirms the page is reachable and, for pricing rows, that the stored amount and SKU name appear on it. It does not re-derive or change the score.","reason":null}}},"whenNotTopPickRules":[{"ifConstraint":"escape_hatch_clause_1","recommendedProviderId":"langsmith","activeModelOrSku":"langsmith-plus","reason":"LangGraph-native node/edge trajectory assertions and state-graph replay — LangSmith"},{"ifConstraint":"exit:self_host_or_open_weights","recommendedProviderId":"arize-phoenix","activeModelOrSku":"arize-phoenix-standard","reason":"self-hosted OpenTelemetry span & tool-call evaluators — Arize Phoenix"},{"ifConstraint":"escape_hatch_clause_3","recommendedProviderId":"langfuse","activeModelOrSku":"langfuse-cloud-core","reason":"open-source MIT tracing + dataset runs — Langfuse"},{"ifConstraint":"exit:self_host_or_open_weights","recommendedProviderId":"promptfoo","activeModelOrSku":"promptfoo-oss","reason":"local CLI tool-call assertions in CI without SaaS — Promptfoo."}],"bootstrap":{"authType":"api_key","envVars":["BRAINTRUST_API_KEY"],"signupUrl":"https://www.braintrust.dev/signup","apiKeyUrl":"https://www.braintrust.dev/app/settings?subroute=api-keys","docsUrl":"https://www.braintrust.dev/docs/start","mcpServer":"npx -y @braintrust/mcp-server","cliInit":null,"sdkInstall":{"npm":"npm install braintrust","pip":"pip install braintrust"},"installCommand":"npm install braintrust","activeModelOrSku":"braintrust-pro","canonicalSnippet":"import { Eval } from \"braintrust\";\nEval(\"agent-eval\", { data: () => [{ input: \"hi\", expected: \"hi\" }], task: async (x) => x, scores: [] });","verifyCommand":"curl -sS https://api.braintrust.dev/v1/project -H \"Authorization: Bearer $BRAINTRUST_API_KEY\""},"quickstartSpec":{"installCommand":"npm install braintrust","envVars":["BRAINTRUST_API_KEY"],"activeModelOrSku":"braintrust-pro","canonicalSnippet":"import { Eval } from \"braintrust\";\nEval(\"agent-eval\", { data: () => [{ input: \"hi\", expected: \"hi\" }], task: async (x) => x, scores: [] });","sdkInstall":{"npm":"npm install braintrust","pip":"pip install braintrust"},"verifyCommand":"curl -sS https://api.braintrust.dev/v1/project -H \"Authorization: Bearer $BRAINTRUST_API_KEY\"","apiKeyUrl":"https://www.braintrust.dev/app/settings?subroute=api-keys","docsUrl":"https://www.braintrust.dev/docs/start","mcpServer":"npx -y @braintrust/mcp-server","bootstrapReason":null},"operationalGotchas":[{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Broader tracing competitors exist","sourceDate":"2026-10-28"},{"code":"hidden_spend_tier_gates","severity":"warning","summary":"Price entry higher than hobby tools","sourceDate":"2026-10-28"}],"relatedPacks":[]}