diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 61ad998e4b3..8f89f28e07c 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -459,6 +459,10 @@ async def get_auto_router_recommendation( Literal["economy", "balanced", "high", "max"], Query(description="How close every admitted model must be to the best available quality score"), ] = "balanced", + optimize_for: Annotated[ + Literal["cost", "task_completion_speed", "balanced"], + Query(description="What to minimize after the selected quality gate is applied"), + ] = "balanced", team_id: Annotated[str | None, Query(description="Team whose model access the recommendation must use")] = None, ) -> AutoRouterRecommendationResponse: """Build an editable four-tier router from only the model groups this caller can use.""" @@ -502,6 +506,7 @@ async def get_auto_router_recommendation( snapshot=snapshot, available_model_deployments=available_deployments, quality_level=quality_level, + optimize_for=optimize_for, ) except ValueError as exc: raise HTTPException( @@ -519,6 +524,7 @@ async def get_auto_router_recommendation( ) return AutoRouterRecommendationResponse( quality_level=quality_level, + optimize_for=optimize_for, snapshot_id=snapshot.snapshot_id, snapshot_generated_at=snapshot.generated_at, available_model_group_count=len(available_deployments), diff --git a/litellm/router.py b/litellm/router.py index f33dfbba7bf..273fce66472 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7865,6 +7865,12 @@ class Router: # WS session wrappers fire with result=None; per-turn costs tracked by inner calls. if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"): return + await self._record_auto_setup_response_latency( + kwargs=kwargs, + completion_response=completion_response, + start_time=start_time, + end_time=end_time, + ) standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None) if standard_logging_object is None: raise ValueError("standard_logging_object is None") @@ -7976,6 +7982,56 @@ class Router: "litellm.router.Router::deployment_callback_on_success(): Exception occured - %s", e ) + async def _record_auto_setup_response_latency( + self, + *, + kwargs: Mapping[str, object], + completion_response: object, + start_time: object, + end_time: object, + ) -> None: + try: + metadata: Final = _SESSION_ADAPTER.validate_python( + get_litellm_metadata_from_kwargs( + dict(kwargs) # mutable-ok: legacy metadata helper requires a concrete dict copy + ) + ) + decision: Final = _SESSION_ADAPTER.validate_python(metadata.get("routing_decision")) + except ValidationError: + return + if decision.get("auto_setup_selection_mode") != "runtime_response_latency": + return + router_model_name = decision.get("router_model_name") + tier = decision.get("tier") + routed_model = decision.get("routed_model") + if not isinstance(router_model_name, str) or not router_model_name: + return + if not isinstance(tier, str) or not tier: + return + if not isinstance(routed_model, str) or not routed_model: + return + registered: Final = self.complexity_routers.get(router_model_name) + if not registered or not any(tagged.strategy.config.auto_setup is not None for tagged in registered): + return + + from litellm.router_strategy.complexity_router.response_latency import ( + record_runtime_response_latency, + ) + + try: + await record_runtime_response_latency( + router_cache=self.cache, + router_model_name=router_model_name, + tier=tier, + routed_model=routed_model, + kwargs=kwargs, + response_obj=completion_response, + start_time=start_time, + end_time=end_time, + ) + except Exception as exc: # noqa: BLE001 -- optional telemetry must never fail a successful completion + verbose_router_logger.warning("Could not record Auto setup response latency: %s", exc) + def sync_deployment_callback_on_success( self, kwargs, # kwargs to completion diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 39078c2f304..d36ce9c2e9a 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -71,31 +71,43 @@ still resolve to a deployment in `model_list`; this configuration does not creat ### Zero-setup Auto configuration The dashboard's Add Router flow can generate the four tiers from the model groups the -current admin or team can actually use. Auto setup asks for one product choice: Quality -level (Economy, Balanced, High, or Max). +current admin or team can actually use. Auto setup asks for two product choices: + +- Quality level: Economy, Balanced, High, or Max +- Optimize for: Cost, Task completion speed, or Balanced The quality level is a gate, not an optimization weight. For every complexity tier, LiteLLM finds the best available model's conservative LiveBench quality score and admits only models within the selected maximum regret: 15 percentage points for Economy, 7 for -Balanced, 3 for High, and 1 for Max. Auto then chooses the lowest estimated cost per -completed task among the admitted models. If none of those models has comparable token -pricing, Auto chooses the highest conservative quality score instead of failing setup. +Balanced, 3 for High, and 1 for Max. The objective then ranks only those admitted models. This means Max still sends trivial work to a smaller model when that model is within one point of the best available quality for trivial work. -Cost is estimated dollars per completed task, not input or output token price. LiteLLM -combines the benchmark request's token mix and completion probability with the configured -deployment's input, cache-read, and output rates. When a model group has multiple -deployments, Auto uses the most conservative estimate. +Cost is benchmark cost per completed task, not input or output token price. It is observed +request dollars divided by quality-adjusted completed questions in the same complexity +bucket, with partial benchmark credit counted as a fractional completion. -Auto selects one model group for each tier and returns a normal, editable `heuristic_v2` -configuration. It does not add a request-time policy, collect live latency observations, -or change how the complexity router runs after creation. Existing routers, manual -configurations, and LiteLLM's deployment-level routing are therefore unchanged. +Task completion speed uses two forms of evidence: -The bundled benchmark data is a versioned snapshot. Creating or regenerating a router uses -the snapshot shipped with that LiteLLM version; the saved router contains only the four -selected tiers and ordinary complexity-router settings. +- SIMPLE and MEDIUM tiers learn equal-output response time from this proxy's successful + requests: time to first token plus the time to generate the expected number of visible + output tokens. Reasoning tokens are excluded. The policy explores every quality-admitted + model twice before using a rolling seven-day sample +- COMPLEX and REASONING tiers use retry-adjusted Terminal-Bench wall-clock time to a + successful task. Results are compared only inside the same agent and benchmark cohort + and fall back to cost when comparable speed evidence is unavailable + +Balanced minimizes the combined log-normalized cost and completion-time scores after the +same quality gate. The recommendation stores its snapshot ID, objective, admitted models, +and rank order directly in the router configuration, so the generated result is visible +and editable. Changing a generated tier's model pool in the dashboard removes that Auto +selection policy and returns the tier to normal manual pool selection. + +These rules apply only when a router is created through Auto setup. Existing routers, +manual configurations, and LiteLLM's deployment-level latency routing are unchanged. The +bundled benchmark data is a versioned snapshot: creating or regenerating a router uses the +snapshot shipped with that LiteLLM version, while an already saved router keeps its stored +policy. ### Heuristic v2 diff --git a/litellm/router_strategy/complexity_router/artifacts/auto_router_snapshot_v0.json b/litellm/router_strategy/complexity_router/artifacts/auto_router_snapshot_v0.json index 7be71f4b34d..67f478713b5 100644 --- a/litellm/router_strategy/complexity_router/artifacts/auto_router_snapshot_v0.json +++ b/litellm/router_strategy/complexity_router/artifacts/auto_router_snapshot_v0.json @@ -1,8 +1,12 @@ { "activation_scope": { + "latency_objectives": [ + "task_completion_speed", + "balanced" + ], "setup_mode": "auto" }, - "artifact_sha256": "e191b5f7225e4da7a178eb5fe8780bcea24fb0166be23e2069338c6fe5522cd5", + "artifact_sha256": "bab7488212d40d96c5e43f80893c5ca2961d0e25160cb1aebbef36a2422b8a2b", "complexity_policy": { "cumulative_shares": { "complex": 1.0, @@ -10,21 +14,36 @@ "standard": 0.75, "trivial": 0.25 }, - "method": "relative_quantiles" + "method": "relative_quantiles", + "online_response_latency_formula": "time_to_first_token_ms + expected_visible_output_tokens / output_tokens_per_ms", + "speed_metric_by_complexity": { + "complex": "offline_task_completion_latency", + "simple": "online_response_latency", + "standard": "offline_task_completion_latency", + "trivial": "online_response_latency" + } }, "definitions": { "cost": "Provider-adjustable estimate of dollars per completed task using the benchmark trajectory's observed token mix and completion rate.", + "online_response_latency": "Time to first token plus expected visible output generation time from LiteLLM's live observations.", "quality": "Mean LiveBench subtask score within the inferred complexity bucket.", - "quality_lower_bound": "Mean subtask score minus 1.96 standard errors, floored at zero." + "quality_lower_bound": "Mean subtask score minus 1.96 standard errors, floored at zero.", + "task_completion_speed": "Terminal-Bench expected wall-clock time to a successful task, including failed attempts." }, "generated_at": "2026-09-03T20:00:00Z", "limitations": [ - "The snapshot is an offline setup prior; it does not change runtime routing, endpoint health checks, or deployment selection.", + "The snapshot is an offline prior; it does not replace live endpoint health or deployment latency routing.", + "Terminal-Bench duration is full agent-task wall time and is only compared inside the same agent/version cohort.", + "Trivial and simple requests require online response-latency evidence; long-horizon benchmark time is never used to rank them.", + "A reference speed route anchors on the cohort containing the highest-quality measured candidate; it never compares raw seconds across cohorts.", + "When no quality-admitted model has comparable speed evidence, speed and balanced routes explicitly fall back to cost.", "LiveBench cost per completion treats partial benchmark credit as a fractional completion.", "LiveBench input-token means are model-level, not complexity-bucket-specific; output-token means are weighted inside each bucket.", - "Provider-adjusted completion cost scales the observed request cost by the benchmark token mix and source prices; cache discounts and non-token charges can make it approximate.", - "LiteLLM recomputes every quality gate after intersecting the snapshot with the user's configured, routable models.", - "Benchmark variants include their measured reasoning effort; setup preserves required effort parameters when the provider supports them." + "Provider-adjusted LiveBench cost scales the observed request cost by the benchmark token mix and source prices; cache discounts and non-token charges can make it approximate.", + "Provider-adjusted Terminal-Bench cost uses uncached input, cache-read input, and output token means; non-token tool charges are not represented.", + "Terminal-Bench token totals span many calls, so per-call long-context thresholds cannot be reconstructed and are not applied to trajectory totals.", + "Reference routes use the full benchmark catalog; LiteLLM must recompute them after intersecting with the user's configured and capability-compatible models.", + "Benchmark variants include their measured reasoning effort; runtime mapping must not silently claim the score for a different effort setting." ], "models": { "claude-fable-5-1-max-effort": { @@ -112,6 +131,96 @@ "source_output_cost_per_token": 5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-fable-5-1-max-effort", + "cohort_id": "claude-code@2.1.257", + "complexity": "complex", + "cost_per_completed_task_usd": 170.09694079545451, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 22.012545279411764, + "mean_cache_read_input_tokens": 5441945.505882353, + "mean_output_tokens": 161621.48235294118, + "mean_uncached_input_tokens": 1034091.7176470588, + "observed_duration_p50_ms": 2313576.159, + "observed_duration_p95_ms": 19944972.068, + "quality_lower_bound": 0.07382276936875576, + "retry_adjusted_completion_ms": 36630585.75418182, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.12941176470588237, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-fable-5-1" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-fable-5-1-max-effort", + "cohort_id": "claude-code@2.1.257", + "complexity": "simple", + "cost_per_completed_task_usd": 25.25749088095238, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 18.720257947058823, + "mean_cache_read_input_tokens": 7568509.458823529, + "mean_output_tokens": 220892.04705882352, + "mean_uncached_input_tokens": 538071.4941176471, + "observed_duration_p50_ms": 2841125.8109999998, + "observed_duration_p95_ms": 9005283.476, + "quality_lower_bound": 0.6390752485008677, + "retry_adjusted_completion_ms": 5309329.376857143, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.7411764705882353, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-fable-5-1" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-fable-5-1-max-effort", + "cohort_id": "claude-code@2.1.257", + "complexity": "standard", + "cost_per_completed_task_usd": 27.663567212962963, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 18.672907868750002, + "mean_cache_read_input_tokens": 8005376.575, + "mean_output_tokens": 218323.6875, + "mean_uncached_input_tokens": 483923.825, + "observed_duration_p50_ms": 2673098.842, + "observed_duration_p95_ms": 7985724.741, + "quality_lower_bound": 0.5664032172496735, + "retry_adjusted_completion_ms": 4729345.715351852, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.675, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-fable-5-1" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-fable-5-1-max-effort", + "cohort_id": "claude-code@2.1.257", + "complexity": "trivial", + "cost_per_completed_task_usd": 20.434632119047624, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 16.092272793750002, + "mean_cache_read_input_tokens": 9009226.35, + "mean_output_tokens": 164275.2875, + "mean_uncached_input_tokens": 546191.3375, + "observed_duration_p50_ms": 2008713.561, + "observed_duration_p95_ms": 11603729.906000001, + "quality_lower_bound": 0.6857763938182242, + "retry_adjusted_completion_ms": 4635716.655238096, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.7875, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-fable-5-1" + } + ] } }, "claude-fable-5-max-effort": { @@ -200,6 +309,96 @@ "source_output_cost_per_token": 5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-fable-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "complex", + "cost_per_completed_task_usd": 392.82553275, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 23.107384279411765, + "mean_cache_read_input_tokens": 8784759.376470588, + "mean_output_tokens": 149270.90588235293, + "mean_uncached_input_tokens": 777170.2588235295, + "observed_duration_p50_ms": 2042428.4970000002, + "observed_duration_p95_ms": 17828109.054, + "quality_lower_bound": 0.025384785707111474, + "retry_adjusted_completion_ms": 83197786.8262, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.058823529411764705, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-fable-5" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-fable-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "simple", + "cost_per_completed_task_usd": 33.590369561320756, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 20.94458337352941, + "mean_cache_read_input_tokens": 11229777.952941176, + "mean_output_tokens": 204881.6, + "mean_uncached_input_tokens": 336043.8823529412, + "observed_duration_p50_ms": 3535340.2729999996, + "observed_duration_p95_ms": 9474656.393, + "quality_lower_bound": 0.5172971118468915, + "retry_adjusted_completion_ms": 6346110.95354717, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.6235294117647059, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-fable-5" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-fable-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "standard", + "cost_per_completed_task_usd": 58.93160589516129, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 22.835997284375, + "mean_cache_read_input_tokens": 11889124.975, + "mean_output_tokens": 173329.6125, + "mean_uncached_input_tokens": 402044.525, + "observed_duration_p50_ms": 2997084.142, + "observed_duration_p95_ms": 10159133.293000001, + "quality_lower_bound": 0.2882439968827771, + "retry_adjusted_completion_ms": 9369974.166258065, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.3875, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-fable-5" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-fable-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "trivial", + "cost_per_completed_task_usd": 29.201946306034483, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 21.171411071875, + "mean_cache_read_input_tokens": 11396242.775, + "mean_output_tokens": 183132.1625, + "mean_uncached_input_tokens": 446857.5875, + "observed_duration_p50_ms": 3460185.4905000003, + "observed_duration_p95_ms": 10187401.28, + "quality_lower_bound": 0.618557347158301, + "retry_adjusted_completion_ms": 5933162.700517242, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.725, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-fable-5" + } + ] } }, "claude-opus-4-5-20251101-thinking-64k-high-effort": { @@ -281,6 +480,12 @@ "source_output_cost_per_token": 2.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "claude-opus-4-6-thinking-auto-high-effort": { @@ -365,6 +570,12 @@ "source_output_cost_per_token": 2.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "claude-opus-4-7-xhigh-effort": { @@ -456,6 +667,12 @@ "source_output_cost_per_token": 2.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "claude-opus-4-8-max-effort": { @@ -549,6 +766,96 @@ "source_output_cost_per_token": 2.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-opus-4-8-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "complex", + "cost_per_completed_task_usd": 658.821080875, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 15.501672491176471, + "mean_cache_read_input_tokens": 14129125.54117647, + "mean_output_tokens": 198919.23529411765, + "mean_uncached_input_tokens": 554347.0235294118, + "observed_duration_p50_ms": 2628999.225, + "observed_duration_p95_ms": 17801145.046, + "quality_lower_bound": 0.006476375454219535, + "retry_adjusted_completion_ms": 210516238.57050002, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.023529411764705882, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-opus-4-8" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-opus-4-8-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "simple", + "cost_per_completed_task_usd": 80.75515165909091, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 20.901333370588237, + "mean_cache_read_input_tokens": 20194287.02352941, + "mean_output_tokens": 302960.2117647059, + "mean_uncached_input_tokens": 516850.5411764706, + "observed_duration_p50_ms": 4935654.733, + "observed_duration_p95_ms": 10689830.749, + "quality_lower_bound": 0.1775797328865609, + "retry_adjusted_completion_ms": 20200870.892045453, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.25882352941176473, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-opus-4-8" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-opus-4-8-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "standard", + "cost_per_completed_task_usd": 193.26893900000002, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 16.9110321625, + "mean_cache_read_input_tokens": 14685576.6375, + "mean_output_tokens": 236491.775, + "mean_uncached_input_tokens": 585015.2125, + "observed_duration_p50_ms": 3971969.2490000003, + "observed_duration_p95_ms": 8566031.918000001, + "quality_lower_bound": 0.043031454647439876, + "retry_adjusted_completion_ms": 48481955.61957143, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0875, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-opus-4-8" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-opus-4-8-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "trivial", + "cost_per_completed_task_usd": 43.279292521276595, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 25.42658435625, + "mean_cache_read_input_tokens": 25549236.875, + "mean_output_tokens": 340427.675, + "mean_uncached_input_tokens": 661020.725, + "observed_duration_p50_ms": 6080651.175, + "observed_duration_p95_ms": 12414037.945, + "quality_lower_bound": 0.47803841126883023, + "retry_adjusted_completion_ms": 10616437.123489361, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.5875, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-opus-4-8" + } + ] } }, "claude-opus-5-max-effort": { @@ -640,6 +947,96 @@ "source_output_cost_per_token": 2.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-opus-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "complex", + "cost_per_completed_task_usd": 281.3378472, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 16.549285129411764, + "mean_cache_read_input_tokens": 15744333.729411764, + "mean_output_tokens": 173575.88235294117, + "mean_uncached_input_tokens": 694124.9764705882, + "observed_duration_p50_ms": 2207406.744, + "observed_duration_p95_ms": 17664645.654, + "quality_lower_bound": 0.025384785707111474, + "retry_adjusted_completion_ms": 94611233.12079999, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.058823529411764705, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-opus-5" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-opus-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "simple", + "cost_per_completed_task_usd": 30.785845800847458, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 21.36899885, + "mean_cache_read_input_tokens": 22501503.37647059, + "mean_output_tokens": 253603.1294117647, + "mean_uncached_input_tokens": 604559.9529411765, + "observed_duration_p50_ms": 3380587.579, + "observed_duration_p95_ms": 15045314.204, + "quality_lower_bound": 0.5895400497870746, + "retry_adjusted_completion_ms": 7657584.832338983, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.6941176470588235, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-opus-5" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-opus-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "standard", + "cost_per_completed_task_usd": 35.05708062195122, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 17.96675381875, + "mean_cache_read_input_tokens": 20685562.3125, + "mean_output_tokens": 191914.2625, + "mean_uncached_input_tokens": 451568.2, + "observed_duration_p50_ms": 3494617.0105, + "observed_duration_p95_ms": 7178773.611, + "quality_lower_bound": 0.4049314460370689, + "retry_adjusted_completion_ms": 7639667.404512195, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.5125, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-opus-5" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-opus-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "trivial", + "cost_per_completed_task_usd": 20.582428286764706, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 17.49506404375, + "mean_cache_read_input_tokens": 17807508.9, + "mean_output_tokens": 185969.275, + "mean_uncached_input_tokens": 630809.3875, + "observed_duration_p50_ms": 2546369.6345, + "observed_duration_p95_ms": 10659490.976, + "quality_lower_bound": 0.7558657249889396, + "retry_adjusted_completion_ms": 5383495.158191176, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.85, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-opus-5" + } + ] } }, "claude-sonnet-4-6-thinking-auto-medium-effort": { @@ -731,6 +1128,12 @@ "source_output_cost_per_token": 1.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "claude-sonnet-5-xhigh-effort": { @@ -824,6 +1227,12 @@ "source_output_cost_per_token": 1.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "deepseek-v4-flash": { @@ -917,6 +1326,12 @@ "source_output_cost_per_token": 2.8e-07, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "deepseek-v4-flash-0731": { @@ -1007,6 +1422,12 @@ "source_output_cost_per_token": 2.8e-07, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "deepseek-v4-flash-vision-exp": { @@ -1088,6 +1509,12 @@ "source_output_cost_per_token": 6.6e-07, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "deepseek-v4-pro": { @@ -1180,6 +1607,12 @@ "source_output_cost_per_token": 8.7e-07, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "deepseek-v4-pro-0813": { @@ -1263,6 +1696,12 @@ "source_output_cost_per_token": 8.7e-07, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gemini-3.1-pro-preview-high": { @@ -1347,6 +1786,12 @@ "source_output_cost_per_token": 1.2e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gemini-3.5-flash-high": { @@ -1431,6 +1876,12 @@ "source_output_cost_per_token": 9e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gemini-3.5-flash-lite-high": { @@ -1514,6 +1965,12 @@ "source_output_cost_per_token": 2.5e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gemini-3.6-flash-high": { @@ -1597,6 +2054,12 @@ "source_output_cost_per_token": 7.5e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gemini-3.7-flash-high": { @@ -1681,6 +2144,96 @@ "source_output_cost_per_token": 3.75e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gemini-3.7-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "complex", + "cost_per_completed_task_usd": 295.9275225750001, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.4815002655882368, + "mean_cache_read_input_tokens": 28612753.65882353, + "mean_output_tokens": 116238.91764705883, + "mean_uncached_input_tokens": 1199530.4, + "observed_duration_p50_ms": 648803.339, + "observed_duration_p95_ms": 6388953.142, + "quality_lower_bound": 0.0020797335877093272, + "retry_adjusted_completion_ms": 167815171.641, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.011764705882352941, + "task_count": 17, + "terminal_bench_model_id": "gemini/gemini-3.7-flash" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gemini-3.7-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "simple", + "cost_per_completed_task_usd": 51.97473815625, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 2.445870030882353, + "mean_cache_read_input_tokens": 17370307.94117647, + "mean_output_tokens": 154050.90588235293, + "mean_uncached_input_tokens": 753874.7176470588, + "observed_duration_p50_ms": 1034465.7279999999, + "observed_duration_p95_ms": 2791459.0560000003, + "quality_lower_bound": 0.018449962264952782, + "retry_adjusted_completion_ms": 25638198.509499997, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.047058823529411764, + "task_count": 17, + "terminal_bench_model_id": "gemini/gemini-3.7-flash" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gemini-3.7-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "standard", + "cost_per_completed_task_usd": 87.11567227500001, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 4.355783613750001, + "mean_cache_read_input_tokens": 39359057.1, + "mean_output_tokens": 159510.375, + "mean_uncached_input_tokens": 1074253.9, + "observed_duration_p50_ms": 1048099.63, + "observed_duration_p95_ms": 6347074.634, + "quality_lower_bound": 0.019613237127295572, + "retry_adjusted_completion_ms": 33926894.975499995, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.05, + "task_count": 16, + "terminal_bench_model_id": "gemini/gemini-3.7-flash" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gemini-3.7-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "trivial", + "cost_per_completed_task_usd": 14.627877795535715, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 5.1197572284375, + "mean_cache_read_input_tokens": 45995506.8375, + "mean_output_tokens": 204981.7, + "mean_uncached_input_tokens": 1201883.7875, + "observed_duration_p50_ms": 1198424.5915, + "observed_duration_p95_ms": 6527013.723, + "quality_lower_bound": 0.2545439440982969, + "retry_adjusted_completion_ms": 5402727.119178573, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.35, + "task_count": 16, + "terminal_bench_model_id": "gemini/gemini-3.7-flash" + } + ] } }, "gemini-3.8-flash-high": { @@ -1764,6 +2317,96 @@ "source_output_cost_per_token": 3.75e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gemini-3.8-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "complex", + "cost_per_completed_task_usd": 0.0, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.0, + "mean_cache_read_input_tokens": 47160828.941176474, + "mean_output_tokens": 178142.89411764705, + "mean_uncached_input_tokens": 1274647.8823529412, + "observed_duration_p50_ms": 989905.341, + "observed_duration_p95_ms": 14753648.89, + "quality_lower_bound": 0.025384785707111474, + "retry_adjusted_completion_ms": 41792359.309, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.058823529411764705, + "task_count": 17, + "terminal_bench_model_id": "gemini/gemini-3.8-flash" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gemini-3.8-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "simple", + "cost_per_completed_task_usd": 0.0, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.0, + "mean_cache_read_input_tokens": 28410489.37647059, + "mean_output_tokens": 182701.0823529412, + "mean_uncached_input_tokens": 972770.9411764706, + "observed_duration_p50_ms": 1238644.2179999999, + "observed_duration_p95_ms": 2836649.066, + "quality_lower_bound": 0.1193204211030341, + "retry_adjusted_completion_ms": 7225142.85575, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.18823529411764706, + "task_count": 17, + "terminal_bench_model_id": "gemini/gemini-3.8-flash" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gemini-3.8-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "standard", + "cost_per_completed_task_usd": 0.0, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.0, + "mean_cache_read_input_tokens": 46424741.4, + "mean_output_tokens": 203380.9375, + "mean_uncached_input_tokens": 1252358.625, + "observed_duration_p50_ms": 1398407.2505, + "observed_duration_p95_ms": 5482899.748, + "quality_lower_bound": 0.02698897769798554, + "retry_adjusted_completion_ms": 28856834.1502, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0625, + "task_count": 16, + "terminal_bench_model_id": "gemini/gemini-3.8-flash" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gemini-3.8-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.0, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.0, + "mean_cache_read_input_tokens": 81941825.4875, + "mean_output_tokens": 263928.3875, + "mean_uncached_input_tokens": 1719578.9125, + "observed_duration_p50_ms": 1614114.02, + "observed_duration_p95_ms": 5601424.65, + "quality_lower_bound": 0.3574781238878701, + "retry_adjusted_completion_ms": 5122828.089918919, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.4625, + "task_count": 16, + "terminal_bench_model_id": "gemini/gemini-3.8-flash" + } + ] } }, "glm-5.2": { @@ -1854,6 +2497,12 @@ "source_output_cost_per_token": 4.4e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "glm-5.3": { @@ -1936,6 +2585,12 @@ "source_output_cost_per_token": 4.4e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "glm-5.3-flash": { @@ -2017,6 +2672,12 @@ "source_output_cost_per_token": 5e-07, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gpt-5.2-2025-12-11-high": { @@ -2099,6 +2760,12 @@ "source_output_cost_per_token": 1.4e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gpt-5.2-codex": { @@ -2181,6 +2848,12 @@ "source_output_cost_per_token": 1.4e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gpt-5.4-mini-xhigh": { @@ -2264,6 +2937,12 @@ "source_output_cost_per_token": 4.5e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gpt-5.4-nano-xhigh": { @@ -2347,6 +3026,12 @@ "source_output_cost_per_token": 1.25e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gpt-5.4-xhigh": { @@ -2436,6 +3121,12 @@ "source_output_cost_per_token": 1.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gpt-5.5-xhigh": { @@ -2522,6 +3213,12 @@ "source_output_cost_per_token": 3e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "gpt-5.6-luna-max": { @@ -2610,6 +3307,96 @@ "source_output_cost_per_token": 6e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-luna-max", + "cohort_id": "codex@0.149.1", + "complexity": "complex", + "cost_per_completed_task_usd": null, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.6297226068235294, + "mean_cache_read_input_tokens": 21967651.752941176, + "mean_output_tokens": 95748.37647058823, + "mean_uncached_input_tokens": 309517.24705882353, + "observed_duration_p50_ms": 812183.1919999999, + "observed_duration_p95_ms": 30386749.601, + "quality_lower_bound": 0.0, + "retry_adjusted_completion_ms": null, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-luna" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-luna-max", + "cohort_id": "codex@0.149.1", + "complexity": "simple", + "cost_per_completed_task_usd": 3.6673198505882363, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.7334639701176473, + "mean_cache_read_input_tokens": 21637394.447058823, + "mean_output_tokens": 168592.02352941176, + "mean_uncached_input_tokens": 397313.6705882353, + "observed_duration_p50_ms": 1451894.867, + "observed_duration_p95_ms": 9231407.450000001, + "quality_lower_bound": 0.12878892241565565, + "retry_adjusted_completion_ms": 14908957.570411764, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.2, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-luna" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-luna-max", + "cohort_id": "codex@0.149.1", + "complexity": "standard", + "cost_per_completed_task_usd": 16.9274472, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.63477927, + "mean_cache_read_input_tokens": 18678964.6875, + "mean_output_tokens": 130148.925, + "mean_uncached_input_tokens": 422307.325, + "observed_duration_p50_ms": 1577440.2845, + "observed_duration_p95_ms": 5629013.643, + "quality_lower_bound": 0.012834336300788755, + "retry_adjusted_completion_ms": 54544322.523, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0375, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-luna" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-luna-max", + "cohort_id": "codex@0.149.1", + "complexity": "trivial", + "cost_per_completed_task_usd": 4.926968522203215, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 2.2787229415189874, + "mean_cache_read_input_tokens": null, + "mean_output_tokens": null, + "mean_uncached_input_tokens": null, + "observed_duration_p50_ms": 3137410.6835000003, + "observed_duration_p95_ms": 28950832.13, + "quality_lower_bound": 0.3574781238878701, + "retry_adjusted_completion_ms": 16030999.12881081, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.4625, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-luna" + } + ] } }, "gpt-5.6-sol-max": { @@ -2697,6 +3484,96 @@ "source_output_cost_per_token": 3e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-sol-max", + "cohort_id": "codex@0.149.1", + "complexity": "complex", + "cost_per_completed_task_usd": 254.19713079999997, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 8.971663439999999, + "mean_cache_read_input_tokens": 16735234.835294118, + "mean_output_tokens": 62267.282352941176, + "mean_uncached_input_tokens": 209666.14117647058, + "observed_duration_p50_ms": 1013924.23, + "observed_duration_p95_ms": 18501705.555, + "quality_lower_bound": 0.012074823021218326, + "retry_adjusted_completion_ms": 105534731.96500002, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.03529411764705882, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-sol" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-sol-max", + "cohort_id": "codex@0.149.1", + "complexity": "simple", + "cost_per_completed_task_usd": 14.184279841379311, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 4.839342534117647, + "mean_cache_read_input_tokens": 7048592.423529412, + "mean_output_tokens": 66036.84705882354, + "mean_uncached_input_tokens": 140110.48235294118, + "observed_duration_p50_ms": 1373627.158, + "observed_duration_p95_ms": 3545131.2939999998, + "quality_lower_bound": 0.2492177176176207, + "retry_adjusted_completion_ms": 4732196.68475862, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.3411764705882353, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-sol" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-sol-max", + "cohort_id": "codex@0.149.1", + "complexity": "standard", + "cost_per_completed_task_usd": 21.92680297777778, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 7.400296005000001, + "mean_cache_read_input_tokens": 12077936.6375, + "mean_output_tokens": 69831.1875, + "mean_uncached_input_tokens": 236228.9, + "observed_duration_p50_ms": 1280606.867, + "observed_duration_p95_ms": 7725407.675000001, + "quality_lower_bound": 0.24345459812221978, + "retry_adjusted_completion_ms": 5344111.560555556, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.3375, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-sol" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-sol-max", + "cohort_id": "codex@0.149.1", + "complexity": "trivial", + "cost_per_completed_task_usd": 13.002322996874996, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 10.401858397499998, + "mean_cache_read_input_tokens": 18088253.65, + "mean_output_tokens": 89979.0625, + "mean_uncached_input_tokens": 278267.675, + "observed_duration_p50_ms": 1556719.1935, + "observed_duration_p95_ms": 8823486.585, + "quality_lower_bound": 0.6995355220341885, + "retry_adjusted_completion_ms": 3411662.1276875, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.8, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-sol" + } + ] } }, "gpt-5.6-terra-max": { @@ -2785,6 +3662,96 @@ "source_output_cost_per_token": 1.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-terra-max", + "cohort_id": "codex@0.149.1", + "complexity": "complex", + "cost_per_completed_task_usd": 207.23259980000006, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 4.876061171764707, + "mean_cache_read_input_tokens": 16777721.388235293, + "mean_output_tokens": 79003.21176470588, + "mean_uncached_input_tokens": 231868.31764705884, + "observed_duration_p50_ms": 760895.202, + "observed_duration_p95_ms": 17511504.763, + "quality_lower_bound": 0.006476375454219535, + "retry_adjusted_completion_ms": 143084489.17000002, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.023529411764705882, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-terra" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-terra-max", + "cohort_id": "codex@0.149.1", + "complexity": "simple", + "cost_per_completed_task_usd": 17.41710915542328, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.6883289976190476, + "mean_cache_read_input_tokens": null, + "mean_output_tokens": null, + "mean_uncached_input_tokens": null, + "observed_duration_p50_ms": 1243788.852, + "observed_duration_p95_ms": 3643298.026, + "quality_lower_bound": 0.13836125681504122, + "retry_adjusted_completion_ms": 7933672.004000001, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.21176470588235294, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-terra" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-terra-max", + "cohort_id": "codex@0.149.1", + "complexity": "standard", + "cost_per_completed_task_usd": 56.02047092, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.5012794325, + "mean_cache_read_input_tokens": 10359244.35, + "mean_output_tokens": 73774.925, + "mean_uncached_input_tokens": 218093.775, + "observed_duration_p50_ms": 1329513.8054999998, + "observed_duration_p95_ms": 3397796.914, + "quality_lower_bound": 0.02698897769798554, + "retry_adjusted_completion_ms": 23351506.7788, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0625, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-terra" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-terra-max", + "cohort_id": "codex@0.149.1", + "complexity": "trivial", + "cost_per_completed_task_usd": 15.850715830434785, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 9.114161602500001, + "mean_cache_read_input_tokens": 30301997.95, + "mean_output_tokens": 159034.0875, + "mean_uncached_input_tokens": 460952.0875, + "observed_duration_p50_ms": 1951635.4270000001, + "observed_duration_p95_ms": 14613717.367999999, + "quality_lower_bound": 0.465690919708387, + "retry_adjusted_completion_ms": 6175673.676978262, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.575, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-terra" + } + ] } }, "grok-4.3": { @@ -2867,6 +3834,12 @@ "source_output_cost_per_token": 2.5e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "grok-4.5": { @@ -2946,6 +3919,96 @@ "source_output_cost_per_token": 6e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "grok-4.5", + "cohort_id": "grok-build@1.0.5", + "complexity": "complex", + "cost_per_completed_task_usd": null, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 2.71251968, + "mean_cache_read_input_tokens": 3847177.035294118, + "mean_output_tokens": 52782.964705882354, + "mean_uncached_input_tokens": 355004.8941176471, + "observed_duration_p50_ms": 809437.804, + "observed_duration_p95_ms": 17566938.117999997, + "quality_lower_bound": 0.0, + "retry_adjusted_completion_ms": null, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0, + "task_count": 17, + "terminal_bench_model_id": "xai/grok-4.5" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "grok-4.5", + "cohort_id": "grok-build@1.0.5", + "complexity": "simple", + "cost_per_completed_task_usd": 46.48075626666667, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 5.4683242666666665, + "mean_cache_read_input_tokens": null, + "mean_output_tokens": null, + "mean_uncached_input_tokens": null, + "observed_duration_p50_ms": 1772457.824, + "observed_duration_p95_ms": 5691062.943, + "quality_lower_bound": 0.06517287423683975, + "retry_adjusted_completion_ms": 22212528.7113, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.11764705882352941, + "task_count": 17, + "terminal_bench_model_id": "xai/grok-4.5" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "grok-4.5", + "cohort_id": "grok-build@1.0.5", + "complexity": "standard", + "cost_per_completed_task_usd": 130.93046981818182, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.2732617454545454, + "mean_cache_read_input_tokens": 5532003.2, + "mean_output_tokens": 68074.2875, + "mean_uncached_input_tokens": 224299.1, + "observed_duration_p50_ms": 1359035.736, + "observed_duration_p95_ms": 3994916.012, + "quality_lower_bound": 0.0068827488981252315, + "retry_adjusted_completion_ms": 62711641.6255, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.025, + "task_count": 16, + "terminal_bench_model_id": "xai/grok-4.5" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "grok-4.5", + "cohort_id": "grok-build@1.0.5", + "complexity": "trivial", + "cost_per_completed_task_usd": 43.48064595862069, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 15.76173416, + "mean_cache_read_input_tokens": null, + "mean_output_tokens": null, + "mean_uncached_input_tokens": null, + "observed_duration_p50_ms": 2102327.9115000004, + "observed_duration_p95_ms": 28832497.533, + "quality_lower_bound": 0.26570633724259446, + "retry_adjusted_completion_ms": 14499273.850896552, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.3625, + "task_count": 16, + "terminal_bench_model_id": "xai/grok-4.5" + } + ] } }, "grok-4.6": { @@ -3028,6 +4091,96 @@ "source_output_cost_per_token": 6e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "grok-4.6", + "cohort_id": "grok-build@1.0.5", + "complexity": "complex", + "cost_per_completed_task_usd": 80.149236, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.7717287529411765, + "mean_cache_read_input_tokens": 4618074.352941177, + "mean_output_tokens": 71291.23529411765, + "mean_uncached_input_tokens": 195723.57647058825, + "observed_duration_p50_ms": 1115803.779, + "observed_duration_p95_ms": 15627159.765, + "quality_lower_bound": 0.018449962264952782, + "retry_adjusted_completion_ms": 58126086.48225, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.047058823529411764, + "task_count": 17, + "terminal_bench_model_id": "xai/grok-4.6" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "grok-4.6", + "cohort_id": "grok-build@1.0.5", + "complexity": "simple", + "cost_per_completed_task_usd": 39.875287432228916, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 7.505936457831325, + "mean_cache_read_input_tokens": 8020312.847058823, + "mean_output_tokens": 109234.10588235295, + "mean_uncached_input_tokens": 243557.15294117646, + "observed_duration_p50_ms": 1821589.598, + "observed_duration_p95_ms": 4264177.431, + "quality_lower_bound": 0.1193204211030341, + "retry_adjusted_completion_ms": 10645207.7405625, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.18823529411764706, + "task_count": 17, + "terminal_bench_model_id": "xai/grok-4.6" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "grok-4.6", + "cohort_id": "grok-build@1.0.5", + "complexity": "standard", + "cost_per_completed_task_usd": 472.3057235443037, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 11.807643088607593, + "mean_cache_read_input_tokens": 12985204.8, + "mean_output_tokens": 93418.95, + "mean_uncached_input_tokens": 287172.2375, + "observed_duration_p50_ms": 1404832.459, + "observed_duration_p95_ms": 7392303.36, + "quality_lower_bound": 0.0068827488981252315, + "retry_adjusted_completion_ms": 74351417.9385, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.025, + "task_count": 16, + "terminal_bench_model_id": "xai/grok-4.6" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "grok-4.6", + "cohort_id": "grok-build@1.0.5", + "complexity": "trivial", + "cost_per_completed_task_usd": 38.115191288888894, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 21.4397951, + "mean_cache_read_input_tokens": 22056990.4, + "mean_output_tokens": 122500.6375, + "mean_uncached_input_tokens": 391925.475, + "observed_duration_p50_ms": 1962060.6230000001, + "observed_duration_p95_ms": 9825563.117, + "quality_lower_bound": 0.45340937868656955, + "retry_adjusted_completion_ms": 4988289.024244444, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.5625, + "task_count": 16, + "terminal_bench_model_id": "xai/grok-4.6" + } + ] } }, "grok-build-0.1": { @@ -3107,6 +4260,12 @@ "source_output_cost_per_token": 2e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "inkling-xhigh": { @@ -3190,6 +4349,12 @@ "source_output_cost_per_token": 4.68e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "kimi-k2.6-thinking": { @@ -3267,6 +4432,12 @@ "source_output_cost_per_token": 4e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "kimi-k2.7-code": { @@ -3356,6 +4527,12 @@ "source_output_cost_per_token": 4e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "kimi-k3": { @@ -3441,6 +4618,12 @@ "source_output_cost_per_token": 1.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "minimax-m3": { @@ -3527,6 +4710,12 @@ "source_output_cost_per_token": 1.2e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "muse-spark-1.1-xhigh": { @@ -3608,6 +4797,12 @@ "source_output_cost_per_token": 4.25e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "muse-spark-1.2-xhigh": { @@ -3689,6 +4884,12 @@ "source_output_cost_per_token": 4.25e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "muse-spark-1.3-xhigh": { @@ -3770,6 +4971,12 @@ "source_output_cost_per_token": 4.25e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "nemotron-3-ultra-550b-a55b": { @@ -3849,6 +5056,12 @@ "source_output_cost_per_token": 2.4e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "ox-alpha-max": { @@ -3926,6 +5139,12 @@ "source_output_cost_per_token": 0.0, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "qwen3.6-27b": { @@ -4009,6 +5228,12 @@ "source_output_cost_per_token": 3.6000000000000003e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "qwen3.6-plus": { @@ -4089,6 +5314,12 @@ "source_output_cost_per_token": 3e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "qwen3.7-max": { @@ -4173,6 +5404,12 @@ "source_output_cost_per_token": 7.5e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "qwen3.8-27b": { @@ -4254,6 +5491,12 @@ "source_output_cost_per_token": 6.5e-07, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "qwen3.8-flash-next": { @@ -4331,6 +5574,12 @@ "source_output_cost_per_token": 4.6999999999999995e-07, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "qwen3.8-max": { @@ -4415,6 +5664,12 @@ "source_output_cost_per_token": 6e-06, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } }, "smaug-agentic": { @@ -4492,6 +5747,12 @@ "source_output_cost_per_token": 1.5e-05, "subtask_count": 5 } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] } } }, @@ -4520,12 +5781,145 @@ "path": "livebench/table_2026_06_25.csv", "sha256": "b8e6339f7fafe6df8f6ecc8b0abe3dc529b5737a6c7edd3e24ffcb36e02fd118", "source_id": "livebench-2026-06-25" + }, + { + "bytes": 15970, + "path": "terminal-bench/submissions/2026-08-26-anthropic-claude-fable-5-max-claude-code.json", + "sha256": "487f60477db9c36bd5cdb3d50c80634233bfa53de3368cf33337a771f17aa07e", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15964, + "path": "terminal-bench/submissions/2026-08-26-anthropic-claude-opus-4-8-max-claude-code.json", + "sha256": "25aea019c65af71083bb684cd26c30c4c6eb3f0c91d4d7473c7090c1b5e1d289", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15958, + "path": "terminal-bench/submissions/2026-08-26-anthropic-claude-opus-5-max-claude-code.json", + "sha256": "06fbebf9bf42c430c49b020f86b9365f5ae772c6063bab4fce98f54e349ed80c", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15932, + "path": "terminal-bench/submissions/2026-08-26-openai-gpt-5-6-luna-max-codex.json", + "sha256": "59ae3614eca2b0c1f087b7a50d65e3fc40dd2ef0188c2fab40d32c5a50f823a7", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15924, + "path": "terminal-bench/submissions/2026-08-26-openai-gpt-5-6-sol-max-codex.json", + "sha256": "b41e5f5f15aeb7657f9d32b33a868aecab6dd6db2d48b7480638455664102c5b", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15932, + "path": "terminal-bench/submissions/2026-08-26-openai-gpt-5-6-terra-max-codex.json", + "sha256": "1c7ca98885e40ed4dc31d23391982848b2c1dd87b0595743fa242ad848f1b670", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15887, + "path": "terminal-bench/submissions/2026-08-26-xai-grok-4-5-none-grok-build.json", + "sha256": "fa618ecb274776479019af6d62c538197cbb563a44d8b8a45752dde51d603b93", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15886, + "path": "terminal-bench/submissions/2026-08-26-xai-grok-4-6-none-grok-build.json", + "sha256": "31526e5dff0e06680085bb5029700b0fddc607962e7c6aea9a911600c6b2df1a", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15943, + "path": "terminal-bench/submissions/2026-09-01-gemini-gemini-3-8-flash-high-mini-swe-agent.json", + "sha256": "835f2090e7d40d9e5c6c5483da49c5f3147a0304e615df49c71aafa859a62c50", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15943, + "path": "terminal-bench/submissions/2026-09-02-anthropic-claude-fable-5-1-max-claude-code.json", + "sha256": "cccdb936c1266db66ce922db48144ed0716c1294eec3dcc1bdcf976711c92813", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15953, + "path": "terminal-bench/submissions/2026-09-02-gemini-gemini-3-7-flash-high-mini-swe-agent.json", + "sha256": "013ace9d0c06b97aed4227b624aff0724257b2ca44a113553a03707545efc0cc", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 405512, + "path": "terminal-bench/trials/claude-fable-5-1-max-claude-code.json", + "sha256": "eace284d655bb870eeb113298b921d1ffde2e53f3c43ecbccf619cc6b2a50dd5", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 405541, + "path": "terminal-bench/trials/claude-fable-5-max-claude-code.json", + "sha256": "f6866893713a1feafe23a2136ae6334ff5cadf6887572e94ca8e33a3cfedc4da", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 406419, + "path": "terminal-bench/trials/claude-opus-4-8-max-claude-code.json", + "sha256": "feddf6a0222d4140696f0e25d72a6186282d20eb9170908692ada19e49a3a9ec", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 404974, + "path": "terminal-bench/trials/claude-opus-5-max-claude-code.json", + "sha256": "7a1bea779168226edfd73b547528a042a26d9fd573a6af7a174f90516ede2494", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 409532, + "path": "terminal-bench/trials/gemini-3-7-flash-high-mini-swe-agent.json", + "sha256": "b674fd778f9148d19cd3e38f441c4ecf168eb0f1056f5d9e75843ed80e66f6d2", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 406835, + "path": "terminal-bench/trials/gemini-3-8-flash-high-mini-swe-agent.json", + "sha256": "791ee2577ac9b9c9100616f53af6df0945d86b835bd39d7318bb7ee15ccf1f73", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 401789, + "path": "terminal-bench/trials/gpt-5-6-luna-max-codex.json", + "sha256": "9e223e1f7b86990d4260b96ccc76044a86b08df7c90b96c9e00f8a2a150134af", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 400386, + "path": "terminal-bench/trials/gpt-5-6-sol-max-codex.json", + "sha256": "68729d3d8bfe09362752a8ac802cbf7fc3bea6768494d9cab14439dbdac1bb54", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 401701, + "path": "terminal-bench/trials/gpt-5-6-terra-max-codex.json", + "sha256": "646b7879615f7ae5a9861bbbd7271416db4734e30a646fa84993e0692a714d6d", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 399870, + "path": "terminal-bench/trials/grok-4-5-grok-build.json", + "sha256": "0745d3a91ec75cd544f8c79ca6b162e74b2a19416d9ce428315604549ff65093", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 399597, + "path": "terminal-bench/trials/grok-4-6-grok-build.json", + "sha256": "6323e4142ebb1c3a054b42340e8cffa7143ff6ddf480dfdcc89cb5a710660178", + "source_id": "terminal-bench-4.0.0" } ], - "manifest_sha256": "32def689d0fcb59afc8ca06fbc0d2f015c1268cb34dd4aaba991e1150ca437e8", + "manifest_sha256": "128c50d897e1d3f5749c5dd9aa4ec8fa5efb9d00dea983af8194a4981c25b557", "source_revisions": { "litellm-model-registry-2026-09-03": "828d561fcc2704eaf90617be8cdc058fccbd8508", - "livebench-2026-06-25": "62240f848c977d4202c1029191ac663498745f2f" + "livebench-2026-06-25": "62240f848c977d4202c1029191ac663498745f2f", + "terminal-bench-4.0.0": "83c7a6172d629c6575b785ab12c8db787bb2e323" } }, "quality_tiers": { @@ -4546,7 +5940,2205 @@ "maximum_quality_regret": 0.01 } }, - "schema_version": "3.0", + "reference_routes": { + "balanced": { + "balanced": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "deepseek-v4-flash-vision-exp", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.5392130921684612, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "deepseek-v4-pro-0813", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "grok-4.6", + "gemini-3.6-flash-high", + "muse-spark-1.3-xhigh", + "claude-opus-5-max-effort", + "kimi-k3", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro-0813", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "glm-5.3", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7797278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.70251553515036, + "selected_model_id": "claude-fable-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.9068668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "cost": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-flash-vision-exp", + "muse-spark-1.3-xhigh", + "qwen3.8-max", + "kimi-k3", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.1042448619785062, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "deepseek-v4-flash-vision-exp", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.5392130921684612, + "selected_model_id": "deepseek-v4-flash-vision-exp", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "grok-4.6", + "gemini-3.6-flash-high", + "muse-spark-1.3-xhigh", + "claude-opus-5-max-effort", + "kimi-k3", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.03819422168482955, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro-0813", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "glm-5.3", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7797278750259673, + "selected_model_id": "deepseek-v4-pro-0813", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "grok-4.6", + "grok-4.5", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max", + "gemini-3.1-pro-preview-high", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.14293474305698176, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.70251553515036, + "selected_model_id": "grok-4.6", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": 0.001468206744000269, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.9068668617803175, + "selected_model_id": "deepseek-v4-flash-0731", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "task_completion_speed": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 36630585.75418182, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "deepseek-v4-flash-vision-exp", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.5392130921684612, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "deepseek-v4-pro-0813", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "grok-4.6", + "gemini-3.6-flash-high", + "muse-spark-1.3-xhigh", + "claude-opus-5-max-effort", + "kimi-k3", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro-0813", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "glm-5.3", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7797278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-max-effort" + ], + "objective_value": 9369974.166258065, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.70251553515036, + "selected_model_id": "claude-fable-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.9068668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + } + }, + "economy": { + "balanced": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.4592130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "kimi-k2.7-code", + "gemini-3.7-flash-high", + "muse-spark-1.1-xhigh", + "qwen3.7-max", + "grok-4.6", + "grok-4.5", + "qwen3.6-plus", + "muse-spark-1.2-xhigh", + "gemini-3.6-flash-high", + "glm-5.2", + "gpt-5.2-2025-12-11-high", + "muse-spark-1.3-xhigh", + "gpt-5.6-luna-max", + "claude-opus-5-max-effort", + "gemini-3.5-flash-high", + "gemini-3.1-pro-preview-high", + "qwen3.8-max", + "gpt-5.2-codex", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.6997278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-opus-5-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-8-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "glm-5.2", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "grok-build-0.1", + "inkling-xhigh", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.6225155351503601, + "selected_model_id": "claude-opus-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "minimax-m3", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "gpt-5.4-nano-xhigh", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "qwen3.6-27b", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-nano-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "minimax-m3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-27b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.8268668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "cost": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "glm-5.3-flash", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "grok-4.5", + "gemini-3.7-flash-high", + "qwen3.8-27b", + "muse-spark-1.3-xhigh", + "muse-spark-1.1-xhigh", + "gemini-3.8-flash-high", + "grok-4.6", + "qwen3.8-max", + "gpt-5.6-terra-max", + "kimi-k3", + "gpt-5.4-xhigh", + "glm-5.3", + "gpt-5.5-xhigh", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.6-sol-max", + "muse-spark-1.2-xhigh", + "claude-opus-5-max-effort", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.08444897731421191, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.4592130921684613, + "selected_model_id": "glm-5.3-flash", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "kimi-k2.7-code", + "gemini-3.7-flash-high", + "muse-spark-1.1-xhigh", + "qwen3.7-max", + "grok-4.6", + "grok-4.5", + "qwen3.6-plus", + "muse-spark-1.2-xhigh", + "gemini-3.6-flash-high", + "glm-5.2", + "gpt-5.2-2025-12-11-high", + "muse-spark-1.3-xhigh", + "gpt-5.6-luna-max", + "claude-opus-5-max-effort", + "gemini-3.5-flash-high", + "gemini-3.1-pro-preview-high", + "qwen3.8-max", + "gpt-5.2-codex", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.0060579263531826775, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.6997278750259673, + "selected_model_id": "deepseek-v4-flash-0731", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "grok-build-0.1", + "deepseek-v4-flash-0731", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "muse-spark-1.1-xhigh", + "grok-4.6", + "grok-4.5", + "muse-spark-1.2-xhigh", + "glm-5.2", + "gemini-3.7-flash-high", + "muse-spark-1.3-xhigh", + "inkling-xhigh", + "gpt-5.2-codex", + "qwen3.8-max", + "gpt-5.2-2025-12-11-high", + "qwen3.7-max", + "gemini-3.1-pro-preview-high", + "gpt-5.4-xhigh", + "gpt-5.6-terra-max", + "gpt-5.5-xhigh", + "kimi-k3", + "gpt-5.6-sol-max", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "glm-5.3", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.0034918518733508376, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "glm-5.2", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "grok-build-0.1", + "inkling-xhigh", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.6225155351503601, + "selected_model_id": "grok-build-0.1", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "minimax-m3", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "gpt-5.4-nano-xhigh", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "qwen3.6-27b", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": 0.001468206744000269, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-nano-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "minimax-m3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-27b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.8268668617803175, + "selected_model_id": "deepseek-v4-flash-0731", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "task_completion_speed": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 36630585.75418182, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.4592130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "kimi-k2.7-code", + "gemini-3.7-flash-high", + "muse-spark-1.1-xhigh", + "qwen3.7-max", + "grok-4.6", + "grok-4.5", + "qwen3.6-plus", + "muse-spark-1.2-xhigh", + "gemini-3.6-flash-high", + "glm-5.2", + "gpt-5.2-2025-12-11-high", + "muse-spark-1.3-xhigh", + "gpt-5.6-luna-max", + "claude-opus-5-max-effort", + "gemini-3.5-flash-high", + "gemini-3.1-pro-preview-high", + "qwen3.8-max", + "gpt-5.2-codex", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.6997278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-opus-5-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-8-max-effort" + ], + "objective_value": 7639667.404512195, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "glm-5.2", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "grok-build-0.1", + "inkling-xhigh", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.6225155351503601, + "selected_model_id": "claude-opus-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "minimax-m3", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "gpt-5.4-nano-xhigh", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "qwen3.6-27b", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-nano-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "minimax-m3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-27b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.8268668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + } + }, + "high": { + "balanced": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5792130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "gpt-5.6-sol-max", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "gpt-5.6-sol-max", + "gpt-5.5-xhigh", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max" + ], + "quality_floor": 0.8197278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.7425155351503601, + "selected_model_id": "claude-fable-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-pro-0813", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "grok-4.3", + "gemini-3.7-flash-high", + "grok-4.6", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "qwen3.7-max", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.9468668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "cost": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 4.002891945931923, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5792130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "gpt-5.6-sol-max", + "gpt-5.5-xhigh", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.18762597508606915, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max" + ], + "quality_floor": 0.8197278750259673, + "selected_model_id": "gpt-5.6-sol-max", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max", + "gemini-3.1-pro-preview-high", + "claude-fable-5-max-effort" + ], + "objective_value": 0.14910282543392475, + "quality_admitted_model_ids": [ + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.7425155351503601, + "selected_model_id": "muse-spark-1.2-xhigh", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "grok-4.3", + "gemini-3.7-flash-high", + "grok-4.6", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "qwen3.7-max", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": 0.012087533779340973, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.9468668617803175, + "selected_model_id": "deepseek-v4-pro-0813", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "task_completion_speed": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 36630585.75418182, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5792130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "gpt-5.6-sol-max", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "gpt-5.6-sol-max", + "gpt-5.5-xhigh", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max" + ], + "quality_floor": 0.8197278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-max-effort" + ], + "objective_value": 9369974.166258065, + "quality_admitted_model_ids": [ + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.7425155351503601, + "selected_model_id": "claude-fable-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-pro-0813", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "grok-4.3", + "gemini-3.7-flash-high", + "grok-4.6", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "qwen3.7-max", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.9468668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + } + }, + "max": { + "balanced": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5992130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "claude-fable-5-1-max-effort", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "quality_floor": 0.8397278750259674, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "fallback_no_speed_evidence", + "fallback_reason": "No quality-admitted model has Terminal-Bench completion-time evidence.", + "objective_eligible_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "objective_value": 0.236816088515293, + "quality_admitted_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7625155351503601, + "selected_model_id": "muse-spark-1.3-xhigh", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "gemini-3.7-flash-high", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "gemini-3.7-flash-high", + "grok-4.6", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.1-pro-preview-high" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6" + ], + "quality_floor": 0.9668668617803176, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "cost": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 4.002891945931923, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5992130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.38505754057873887, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "quality_floor": 0.8397278750259674, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "objective_value": 0.236816088515293, + "quality_admitted_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7625155351503601, + "selected_model_id": "muse-spark-1.3-xhigh", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "gemini-3.7-flash-high", + "grok-4.6", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.1-pro-preview-high" + ], + "objective_value": 0.024636455346966592, + "quality_admitted_model_ids": [ + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6" + ], + "quality_floor": 0.9668668617803176, + "selected_model_id": "gemini-3.7-flash-high", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "task_completion_speed": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 36630585.75418182, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5992130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "claude-fable-5-1-max-effort", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "quality_floor": 0.8397278750259674, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "fallback_no_speed_evidence", + "fallback_reason": "No quality-admitted model has Terminal-Bench completion-time evidence.", + "objective_eligible_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "objective_value": 0.236816088515293, + "quality_admitted_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7625155351503601, + "selected_model_id": "muse-spark-1.3-xhigh", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "gemini-3.7-flash-high", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "gemini-3.7-flash-high", + "grok-4.6", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.1-pro-preview-high" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6" + ], + "quality_floor": 0.9668668617803176, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + } + } + }, + "schema_version": "2.1", "snapshot_id": "2026-09-03-v0", "task_complexities": { "livebench-2026-06-25": { @@ -4573,6 +8165,74 @@ "typescript": "complex", "typos": "simple", "zebra_puzzle": "simple" + }, + "terminal-bench-4.0.0": { + "terminal-bench/atrx-vep-crispr": "trivial", + "terminal-bench/batched-eval-parity": "simple", + "terminal-bench/biped-contact-dynamics": "simple", + "terminal-bench/bun-sourcemap-leak": "complex", + "terminal-bench/cad-model": "simple", + "terminal-bench/cargo-flight-dispatch": "complex", + "terminal-bench/coq-block-bound": "trivial", + "terminal-bench/ctr-optimization": "complex", + "terminal-bench/cumulative-layout-shift": "trivial", + "terminal-bench/data-anonymization": "complex", + "terminal-bench/distributed-dedup": "simple", + "terminal-bench/embedding-drift-monitor": "simple", + "terminal-bench/fin-saccr-rwa": "trivial", + "terminal-bench/foodstuff-beta-activity": "complex", + "terminal-bench/formal-crypto": "standard", + "terminal-bench/fp8-rmsnorm-gemm": "trivial", + "terminal-bench/freecad-impeller": "complex", + "terminal-bench/freecad-platform-drawing": "standard", + "terminal-bench/freecad-spring-clip": "simple", + "terminal-bench/freight-dispatch-shift": "complex", + "terminal-bench/glycan-ms2-elucidation": "complex", + "terminal-bench/gsea-proteomics": "standard", + "terminal-bench/heat-pump-warranty": "standard", + "terminal-bench/hof-topology-interpenetration": "simple", + "terminal-bench/html-js-filter": "standard", + "terminal-bench/interleaved-vigenere": "simple", + "terminal-bench/intrastat-meldung": "trivial", + "terminal-bench/jax-speedrun-gpu": "complex", + "terminal-bench/ks-solver-cpp": "standard", + "terminal-bench/kv-live-surgery": "simple", + "terminal-bench/lake-temp-glm": "complex", + "terminal-bench/layout-config-recreation": "complex", + "terminal-bench/layout-config-recreation2": "trivial", + "terminal-bench/legacy-utility-triage": "standard", + "terminal-bench/live-database-cutover": "standard", + "terminal-bench/math-eval-grader": "standard", + "terminal-bench/medical-claims-processing": "complex", + "terminal-bench/mp-checkpoint-consolidation": "trivial", + "terminal-bench/music-harmony": "complex", + "terminal-bench/mvcc-lsm-compaction": "simple", + "terminal-bench/nextjs-performance": "standard", + "terminal-bench/ontology-kg-querying": "complex", + "terminal-bench/payments-pipeline-fix": "simple", + "terminal-bench/photonic-waveguide-routing": "simple", + "terminal-bench/pretrain-shard-corruption": "standard", + "terminal-bench/production-planning": "standard", + "terminal-bench/protein-autointerp-disulfide": "complex", + "terminal-bench/react-lead-form": "trivial", + "terminal-bench/retro-console-soc": "trivial", + "terminal-bench/risk-scorer-replay": "trivial", + "terminal-bench/roy-polymorph-cn": "complex", + "terminal-bench/rs-archive-clone": "simple", + "terminal-bench/satb-audio-transcription": "standard", + "terminal-bench/session-window-debug": "complex", + "terminal-bench/sglang-qwen-burst": "standard", + "terminal-bench/shadow-relay": "trivial", + "terminal-bench/sound-change-cascade": "simple", + "terminal-bench/takens-embedding-lean": "standard", + "terminal-bench/telecom-entity-resolution": "trivial", + "terminal-bench/uefi-bootkit": "trivial", + "terminal-bench/vba-userform-port": "simple", + "terminal-bench/vf2-speedup-networkx": "simple", + "terminal-bench/vllm-deepseek-streaming": "standard", + "terminal-bench/vpp-loss-divergence": "trivial", + "terminal-bench/wal-recovery-ordering": "simple", + "terminal-bench/wdm-design": "trivial" } } } diff --git a/litellm/router_strategy/complexity_router/auto_setup.py b/litellm/router_strategy/complexity_router/auto_setup.py index 9e41a4d25d8..2c61cf1aa0f 100644 --- a/litellm/router_strategy/complexity_router/auto_setup.py +++ b/litellm/router_strategy/complexity_router/auto_setup.py @@ -13,10 +13,16 @@ from typing import Final, Literal from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator -from .config import ComplexityRouterConfig +from .config import ( + AutoSetupCandidate, + AutoSetupConfig, + AutoSetupObjective, + AutoSetupQualityLevel, + AutoSetupTierPolicy, + ComplexityRouterConfig, +) SnapshotComplexity = Literal["trivial", "simple", "standard", "complex"] -AutoSetupQualityLevel = Literal["economy", "balanced", "high", "max"] _SNAPSHOT_FILENAME: Final = "auto_router_snapshot_v0.json" _TIER_COMPLEXITIES: Final[Mapping[str, SnapshotComplexity]] = MappingProxyType( @@ -27,6 +33,7 @@ _TIER_COMPLEXITIES: Final[Mapping[str, SnapshotComplexity]] = MappingProxyType( "REASONING": "complex", } ) +_EASY_COMPLEXITIES: Final = frozenset(("trivial", "simple")) _VERSION_SEPARATOR: Final = re.compile(r"(?<=\d)\.(?=\d)") _SNAPSHOT_PAYLOAD_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -68,12 +75,36 @@ class SnapshotQualityProfile(BaseModel): subtask_count: int = Field(gt=0) +class SnapshotSpeedProfile(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + attempt_count: int = Field(gt=0) + benchmark_model_id: str + cohort_id: str + complexity: SnapshotComplexity + cost_per_completed_task_usd: float | None = Field(default=None, ge=0) + excluded_infrastructure_trials: int = Field(ge=0) + mean_attempt_cost_usd: float | None = Field(default=None, ge=0) + mean_cache_read_input_tokens: float | None = Field(default=None, ge=0) + mean_output_tokens: float | None = Field(default=None, ge=0) + mean_uncached_input_tokens: float | None = Field(default=None, ge=0) + observed_duration_p50_ms: float = Field(ge=0) + observed_duration_p95_ms: float = Field(ge=0) + quality_lower_bound: float = Field(ge=0, le=1) + retry_adjusted_completion_ms: float | None = Field(default=None, ge=0) + source_id: str + success_probability: float = Field(ge=0, le=1) + task_count: int = Field(gt=0) + terminal_bench_model_id: str + + class SnapshotModel(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) benchmark_model_id: str identity: SnapshotIdentity quality_by_complexity: Mapping[SnapshotComplexity, SnapshotQualityProfile] + task_completion_speed_by_complexity: Mapping[SnapshotComplexity, tuple[SnapshotSpeedProfile, ...]] class AutoRouterSnapshot(BaseModel): @@ -88,7 +119,8 @@ class AutoRouterSnapshot(BaseModel): models: Mapping[str, SnapshotModel] provenance: Mapping[str, object] quality_tiers: Mapping[AutoSetupQualityLevel, SnapshotQualityTier] - schema_version: Literal["3.0"] + reference_routes: Mapping[str, object] + schema_version: Literal["2.1"] snapshot_id: str task_complexities: Mapping[str, Mapping[str, SnapshotComplexity]] @@ -105,12 +137,24 @@ class AutoRouterSnapshot(BaseModel): raise ValueError(f"Auto Router snapshot model identity mismatch for {model_id}") if frozenset(model.quality_by_complexity) != expected_complexities: raise ValueError(f"Auto Router snapshot model {model_id} lacks complete quality evidence") + if frozenset(model.task_completion_speed_by_complexity) != expected_complexities: + raise ValueError(f"Auto Router snapshot model {model_id} lacks complete speed buckets") if model.identity.is_routable != bool(model.identity.litellm_model_keys): raise ValueError(f"Auto Router snapshot routability mismatch for {model_id}") for complexity in expected_complexities: quality = model.quality_by_complexity[complexity] if quality.benchmark_model_id != model_id or quality.complexity != complexity: raise ValueError(f"Auto Router snapshot quality identity mismatch for {model_id}/{complexity}") + for speed in model.task_completion_speed_by_complexity[complexity]: + token_values = ( + speed.mean_uncached_input_tokens, + speed.mean_cache_read_input_tokens, + speed.mean_output_tokens, + ) + if any(value is None for value in token_values) != all(value is None for value in token_values): + raise ValueError(f"Auto Router snapshot has partial token evidence for {model_id}/{complexity}") + if speed.benchmark_model_id != model_id or speed.complexity != complexity: + raise ValueError(f"Auto Router snapshot speed identity mismatch for {model_id}/{complexity}") for alias in model.identity.litellm_model_keys: normalized = _normalized_model_key(alias) owner = seen_aliases.setdefault(normalized, model.identity.family_id) @@ -149,7 +193,7 @@ class AutoSetupDeployment(BaseModel): pricing: AutoSetupDeploymentPricing | None = None -AutoSetupExclusionReason = Literal["no_benchmark_match", "mixed_model_group"] +AutoSetupExclusionReason = Literal["no_benchmark_match", "mixed_model_group", "pricing_unavailable"] @dataclass(frozen=True, slots=True) @@ -162,8 +206,9 @@ class AutoSetupInventoryExclusion: class _AvailableProfile: model_name: str benchmark_model_id: str - cost_per_completed_task_usd: float | None + cost_per_completed_task_usd: float quality: SnapshotQualityProfile + speed: tuple[SnapshotSpeedProfile, ...] required_parameters: Mapping[str, object] @@ -253,7 +298,18 @@ def analyze_auto_setup_inventory( """Return identity-safe groups and explicit reasons for every excluded group.""" groups, exclusions = _resolve_groups(snapshot, available_model_deployments) - return tuple(group.model_name for group in groups), exclusions + eligible: list[str] = [] + mutable_exclusions = list(exclusions) + for group in groups: + if any( + _quality_completion_cost(model.quality_by_complexity[complexity], group.deployments) is not None + for model in group.matches + for complexity in _TIER_COMPLEXITIES.values() + ): + eligible.append(group.model_name) + else: + mutable_exclusions.append(AutoSetupInventoryExclusion(group.model_name, "pricing_unavailable")) + return tuple(eligible), tuple(mutable_exclusions) def _rate_for_input_size( @@ -289,6 +345,19 @@ def _rate_for_input_size( return base +def _base_rate( + pricing: AutoSetupDeploymentPricing, + field: Literal["input", "output", "cache"], +) -> float: + if field == "input": + return pricing.input_cost_per_token + if field == "output": + return pricing.output_cost_per_token + if pricing.cache_read_input_token_cost is not None: + return pricing.cache_read_input_token_cost + return pricing.input_cost_per_token + + def _quality_completion_cost( profile: SnapshotQualityProfile, deployments: Sequence[AutoSetupDeployment], @@ -312,6 +381,31 @@ def _quality_completion_cost( return max(costs, default=None) +def _speed_completion_cost( + speed: SnapshotSpeedProfile, + deployments: Sequence[AutoSetupDeployment], +) -> float | None: + if ( + speed.success_probability <= 0 + or speed.mean_uncached_input_tokens is None + or speed.mean_cache_read_input_tokens is None + or speed.mean_output_tokens is None + ): + return None + costs: list[float] = [] + for deployment in deployments: + pricing = deployment.pricing + if pricing is None: + return None + attempt_cost = ( + speed.mean_uncached_input_tokens * _base_rate(pricing, "input") + + speed.mean_cache_read_input_tokens * _base_rate(pricing, "cache") + + speed.mean_output_tokens * _base_rate(pricing, "output") + ) + costs.append(attempt_cost / speed.success_probability) + return max(costs, default=None) + + def _available_profiles( snapshot: AutoRouterSnapshot, available_model_deployments: Mapping[str, Sequence[AutoSetupDeployment]], @@ -333,12 +427,19 @@ def _available_profiles( ), ) cost = _quality_completion_cost(best.quality_by_complexity[complexity], group.deployments) + if cost is None: + continue + priced_speed = tuple( + speed.model_copy(update={"cost_per_completed_task_usd": _speed_completion_cost(speed, group.deployments)}) + for speed in best.task_completion_speed_by_complexity[complexity] + ) profiles.append( _AvailableProfile( model_name=group.model_name, benchmark_model_id=best.benchmark_model_id, cost_per_completed_task_usd=cost, quality=best.quality_by_complexity[complexity], + speed=priced_speed, required_parameters=best.identity.required_parameters, ) ) @@ -350,8 +451,7 @@ def _cost_rank(profiles: Sequence[_AvailableProfile]) -> tuple[_AvailableProfile sorted( profiles, key=lambda profile: ( - profile.cost_per_completed_task_usd is None, - profile.cost_per_completed_task_usd if profile.cost_per_completed_task_usd is not None else math.inf, + profile.cost_per_completed_task_usd, -profile.quality.quality_lower_bound, profile.model_name, ), @@ -359,15 +459,146 @@ def _cost_rank(profiles: Sequence[_AvailableProfile]) -> tuple[_AvailableProfile ) +def _normalize_log(value: float, values: Sequence[float]) -> float: + logged: Final = tuple(math.log(max(item, 1e-12)) for item in values) + low: Final = min(logged) + high: Final = max(logged) + if low == high: + return 0.0 + return (math.log(max(value, 1e-12)) - low) / (high - low) + + +def _hard_task_rank( + profiles: Sequence[_AvailableProfile], objective: AutoSetupObjective +) -> tuple[tuple[_AvailableProfile, ...], str]: + cost_ranked: Final = _cost_rank(profiles) + speed_profiles: Final = tuple( + (profile, speed) + for profile in profiles + for speed in profile.speed + if speed.retry_adjusted_completion_ms is not None + ) + if not speed_profiles: + return cost_ranked, "fallback_no_speed_evidence" + cohorts: dict[ # mutable-ok: local cohort index never escapes + str, dict[str, tuple[_AvailableProfile, SnapshotSpeedProfile]] + ] = {} # mutable-ok: local cohort index is populated before selection + for profile, speed in speed_profiles: + by_model = cohorts.setdefault( + speed.cohort_id, + {}, # mutable-ok: local cohort index needs an empty per-cohort bucket + ) + current = by_model.get(profile.model_name) + if current is None or speed.attempt_count > current[1].attempt_count: + by_model[profile.model_name] = (profile, speed) + measured_profiles: Final = tuple( + { # mutable-ok: transient dict provides deterministic model-name deduplication + profile.model_name: profile for profile, _ in speed_profiles + }.values() + ) + anchor: Final = max( + measured_profiles, + key=lambda profile: (profile.quality.quality_lower_bound, -profile.cost_per_completed_task_usd), + ) + anchor_cohorts: Final = tuple(cohort_id for cohort_id, by_model in cohorts.items() if anchor.model_name in by_model) + selected_cohort = max( + anchor_cohorts, + key=lambda cohort_id: ( + len(cohorts[cohort_id]), + sum(speed.attempt_count for _, speed in cohorts[cohort_id].values()), + cohort_id, + ), + ) + comparable = tuple(cohorts[selected_cohort].values()) + if objective == "task_completion_speed": + return tuple( + profile + for profile, _ in sorted( + comparable, + key=lambda item: ( + item[1].retry_adjusted_completion_ms or math.inf, + -item[0].quality.quality_lower_bound, + item[0].model_name, + ), + ) + ), "cohort_scoped" + balanced_comparable: list[ # mutable-ok: local typed accumulator is consumed immediately + tuple[_AvailableProfile, float, float] + ] = [] # mutable-ok: local typed accumulator is frozen by the return expression + for profile, speed in comparable: + cost = speed.cost_per_completed_task_usd + completion_ms = speed.retry_adjusted_completion_ms + if cost is not None and completion_ms is not None: + balanced_comparable.append((profile, cost, completion_ms)) + if not balanced_comparable: + return cost_ranked, "fallback_no_comparable_cost_evidence" + costs: Final = tuple(cost for _, cost, _ in balanced_comparable) + times: Final = tuple(completion_ms for _, _, completion_ms in balanced_comparable) + return tuple( + profile + for profile, _, _ in sorted( + balanced_comparable, + key=lambda item: ( + math.hypot( + _normalize_log(item[1], costs), + _normalize_log(item[2], times), + ), + -item[0].quality.quality_lower_bound, + item[0].model_name, + ), + ) + ), "cohort_scoped" + + +def _policy_candidate(profile: _AvailableProfile) -> AutoSetupCandidate: + return AutoSetupCandidate( + model_name=profile.model_name, + benchmark_model_id=profile.benchmark_model_id, + quality_lower_bound=profile.quality.quality_lower_bound, + cost_per_completed_task_usd=profile.cost_per_completed_task_usd, + ) + + +def _tier_policy( + profiles: Sequence[_AvailableProfile], + complexity: SnapshotComplexity, + objective: AutoSetupObjective, +) -> AutoSetupTierPolicy: + cost_ranked: Final = _cost_rank(profiles) + if objective == "cost": + return AutoSetupTierPolicy( + selection_mode="snapshot_ranked", + candidates=tuple(_policy_candidate(profile) for profile in cost_ranked), + evidence_status="direct", + ) + if complexity in _EASY_COMPLEXITIES: + return AutoSetupTierPolicy( + selection_mode="runtime_response_latency", + candidates=tuple(_policy_candidate(profile) for profile in cost_ranked), + cold_start_model=cost_ranked[0].model_name, + evidence_status="runtime_response_speed_required", + ) + ranked, evidence_status = _hard_task_rank(profiles, objective) + return AutoSetupTierPolicy( + selection_mode="snapshot_ranked", + candidates=tuple(_policy_candidate(profile) for profile in ranked), + evidence_status=evidence_status, + ) + + def build_auto_setup_config( *, snapshot: AutoRouterSnapshot, available_model_deployments: Mapping[str, Sequence[AutoSetupDeployment]], quality_level: AutoSetupQualityLevel, + optimize_for: AutoSetupObjective, ) -> ComplexityRouterConfig: """Recompute quality gates and rankings over only the caller's available groups.""" maximum_regret: Final = snapshot.quality_tiers[quality_level].maximum_quality_regret + policies: Final[ # mutable-ok: Pydantic config assembly is local to this builder + dict[str, AutoSetupTierPolicy] + ] = {} # mutable-ok: Pydantic config assembly is local to this builder tiers: Final[dict[str, list[str]]] = {} # mutable-ok: Pydantic config assembly is local to this builder tier_model_configs: Final[ # mutable-ok: Pydantic config assembly is local to this builder dict[str, list[dict[str, object]]] @@ -379,20 +610,21 @@ def build_auto_setup_config( best_quality = max(profile.quality.quality_lower_bound for profile in profiles) floor = max(0.0, best_quality - maximum_regret) admitted = tuple(profile for profile in profiles if profile.quality.quality_lower_bound >= floor) - selected = _cost_rank(admitted)[0] - tiers[tier_name] = [selected.model_name] - parameterized: list[dict[str, object]] = ( - [ # mutable-ok: Pydantic validates and owns these JSON-shaped rows - { # mutable-ok: each row is a Pydantic input payload - "model_name": selected.model_name, - "litellm_params": dict( # mutable-ok: Pydantic needs a concrete JSON-shaped params object - selected.required_parameters - ), - } - ] - if selected.required_parameters - else [] - ) + policy = _tier_policy(admitted, complexity, optimize_for) + policies[tier_name] = policy + tiers[tier_name] = [ # mutable-ok: ComplexityRouterConfig owns this JSON-shaped tier list + candidate.model_name for candidate in policy.candidates + ] + parameterized: list[dict[str, object]] = [ # mutable-ok: Pydantic validates and owns these JSON-shaped rows + { # mutable-ok: each row is a Pydantic input payload + "model_name": profile.model_name, + "litellm_params": dict( # mutable-ok: Pydantic needs a concrete JSON-shaped params object + profile.required_parameters + ), + } + for profile in admitted + if profile.model_name in frozenset(tiers[tier_name]) and profile.required_parameters + ] if parameterized: tier_model_configs[tier_name] = parameterized @@ -401,5 +633,12 @@ def build_auto_setup_config( "tiers": tiers, "tier_model_configs": tier_model_configs, "classifier_type": "heuristic_v2", + "auto_setup": AutoSetupConfig( + snapshot_id=snapshot.snapshot_id, + snapshot_sha256=snapshot.artifact_sha256, + quality_level=quality_level, + optimize_for=optimize_for, + tier_policies=policies, + ).model_dump(mode="json"), } ) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 17e3d1256d0..8f06b0c97f4 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1322,6 +1322,13 @@ class ComplexityRouter(CustomLogger): if tier is not None: tier_name: Final = _tier_name(tier) decision["tier"] = tier_name + if self.config.auto_setup is not None: + auto_policy: Final = self.config.auto_setup.tier_policies.get(tier_name) + if auto_policy is not None: + decision["auto_setup_snapshot_id"] = self.config.auto_setup.snapshot_id + decision["auto_setup_quality_level"] = self.config.auto_setup.quality_level + decision["auto_setup_optimize_for"] = self.config.auto_setup.optimize_for + decision["auto_setup_selection_mode"] = auto_policy.selection_mode if not self.config.has_custom_tiers: label = self.config.tier_label(ComplexityTier(tier_name)) if label != tier_name: @@ -1815,6 +1822,37 @@ class ComplexityRouter(CustomLogger): raise ValueError(f"Empty model pool for tier {tier_key}") return random.choice(model) + async def _pick_from_auto_setup_policy(self, tier_key: str, pool: Sequence[str]) -> str | None: + setup: Final = self.config.auto_setup + if setup is None: + return None + policy: Final = setup.tier_policies.get(tier_key) + if policy is None: + return None + available: Final = tuple(dict.fromkeys(pool)) + if not available: + raise ValueError(f"No models configured for tier {tier_key}") + ranked: Final = tuple( + candidate.model_name for candidate in policy.candidates if candidate.model_name in frozenset(available) + ) + if policy.selection_mode == "snapshot_ranked": + return ranked[0] if ranked else available[0] + + from .response_latency import select_runtime_response_model + + cold_start: Final = policy.cold_start_model + if cold_start is None: + raise ValueError(f"Auto setup runtime response-latency policy for {tier_key} has no cold-start model") + return await select_runtime_response_model( + router_cache=self.litellm_router_instance.cache, + router_model_name=self.model_name, + tier=tier_key, + candidates=policy.candidates, + available_models=available, + cold_start_model=cold_start, + objective=setup.optimize_for, + ) + def _tier_pools(self) -> dict[str, list[str]]: return {tier: (models if isinstance(models, list) else [models]) for tier, models in self.config.tiers.items()} @@ -1826,17 +1864,25 @@ class ComplexityRouter(CustomLogger): request_kwargs: dict, allowed_models: tuple[str, ...] | None = None, ) -> str: + tier_key = _tier_name(tier) if not self.config.plugins: - if allowed_models is not None: - return self._pick_from_tier_value(allowed_models, _tier_name(tier)) - return self.get_model_for_tier(tier) + configured = self.config.tiers.get(tier_key) + if configured is None: + return self.get_model_for_tier(tier) + full_pool = (configured,) if isinstance(configured, str) else tuple(configured) + pool = ( + tuple(model for model in full_pool if model in allowed_models) + if allowed_models is not None + else full_pool + ) + direct_auto_pick: Final = await self._pick_from_auto_setup_policy(tier_key, pool) + return direct_auto_pick if direct_auto_pick is not None else self._pick_from_tier_value(pool, tier_key) from litellm.types.router import RoutingContext - tier_key: Final = _tier_name(tier) metadata_key: Final = get_metadata_variable_name_from_kwargs(request_kwargs) - full_pool: Final = tuple(self._tier_pools().get(tier_key, ())) - pool: Final = ( + full_pool = tuple(self._tier_pools().get(tier_key, ())) + pool = ( tuple(model for model in full_pool if model in allowed_models) if allowed_models is not None else full_pool ) if not pool: @@ -1860,7 +1906,12 @@ class ComplexityRouter(CustomLogger): # silently bypassed. Raise instead, matching the Router-level plugin # pipeline's own fail-closed behavior for the same situation. raise ValueError(f"No candidate models left for tier {tier_key} after routing-plugin filtering") - return self._pick_from_tier_value(context.candidate_models, tier_key) + filtered_auto_pick: Final = await self._pick_from_auto_setup_policy(tier_key, context.candidate_models) + return ( + filtered_auto_pick + if filtered_auto_pick is not None + else self._pick_from_tier_value(context.candidate_models, tier_key) + ) def _ensure_adaptive_router(self) -> Any | None: if not self.config.adaptive: diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 70c1b281e31..e1f78e3d969 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -27,6 +27,81 @@ class ComplexityTier(str, Enum): REASONING = "REASONING" +AutoSetupQualityLevel = Literal["economy", "balanced", "high", "max"] +AutoSetupObjective = Literal["cost", "task_completion_speed", "balanced"] +AutoSetupSelectionMode = Literal["snapshot_ranked", "runtime_response_latency"] + + +class AutoSetupCandidate(BaseModel): + """One benchmark-backed model group in an Auto-generated tier policy.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + model_name: str = Field(min_length=1) + benchmark_model_id: str = Field(min_length=1) + quality_lower_bound: float = Field(ge=0, le=1) + cost_per_completed_task_usd: float = Field(ge=0) + + +class AutoSetupTierPolicy(BaseModel): + """How an Auto-generated tier chooses from its quality-admitted model groups.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + selection_mode: AutoSetupSelectionMode + candidates: tuple[AutoSetupCandidate, ...] = Field(min_length=1) + cold_start_model: str | None = None + evidence_status: str = Field(min_length=1) + + @model_validator(mode="after") + def _validate_candidates(self) -> "AutoSetupTierPolicy": + names: Final = tuple(candidate.model_name for candidate in self.candidates) + if len(names) != len(frozenset(names)): + raise ValueError("Auto setup tier candidates must use distinct model_name values") + if self.selection_mode == "runtime_response_latency": + if self.cold_start_model not in names: + raise ValueError("A runtime response-latency policy needs a cold_start_model in candidates") + elif self.cold_start_model is not None: + raise ValueError("cold_start_model is only valid for runtime_response_latency") + return self + + +class AutoSetupConfig(BaseModel): + """Provenance and selection policy stamped only by zero-setup Auto creation.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["1"] = "1" + snapshot_id: str = Field(min_length=1) + snapshot_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + quality_level: AutoSetupQualityLevel + optimize_for: AutoSetupObjective + tier_policies: Mapping[str, AutoSetupTierPolicy] + + @field_validator("tier_policies", mode="before") + @classmethod + def _freeze_tier_policies(cls, value: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType(dict(value)) + + @field_serializer("tier_policies") + def _serialize_tier_policies(self, value: Mapping[str, AutoSetupTierPolicy]) -> Mapping[str, AutoSetupTierPolicy]: + return dict(value) # mutable-ok: Pydantic JSON serialization requires a concrete mapping + + @model_validator(mode="after") + def _validate_tier_policies(self) -> "AutoSetupConfig": + expected: Final = frozenset(tier.value for tier in TIER_SEVERITY_ORDER) + actual: Final = frozenset(self.tier_policies) + if actual != expected: + raise ValueError( + f"Auto setup tier_policies must define exactly {', '.join(tier.value for tier in TIER_SEVERITY_ORDER)}" + ) + if self.optimize_for == "cost" and any( + policy.selection_mode != "snapshot_ranked" for policy in self.tier_policies.values() + ): + raise ValueError("Cost-optimized Auto setup cannot enable runtime response-latency selection") + return self + + class ClassificationRubric(str, Enum): """Which calibration examples, and for BUSINESS which tier criteria, the built-in classifier rubric carries.""" @@ -517,6 +592,14 @@ class ComplexityRouterConfig(BaseModel): default_factory=dict, ) + auto_setup: AutoSetupConfig | None = Field( + default=None, + description=( + "Benchmark snapshot and per-tier selection policy generated by the zero-setup Auto flow. " + "Absent on manually configured routers, whose pool selection behavior is unchanged" + ), + ) + tier_definitions: tuple[TierDefinition, ...] | None = Field( default=None, description=( @@ -1422,6 +1505,24 @@ class ComplexityRouterConfig(BaseModel): ) return self + @model_validator(mode="after") + def _validate_auto_setup(self) -> "ComplexityRouterConfig": + setup: Final = self.auto_setup + if setup is None: + return self + if self.adaptive: + raise ValueError("auto_setup and adaptive=True cannot both select from the same tier pool") + if self.tier_definitions is not None: + raise ValueError("auto_setup requires the built-in four-tier complexity ladder") + for tier_name, policy in setup.tier_policies.items(): + configured = self.tiers.get(tier_name) + configured_names = frozenset((configured,)) if isinstance(configured, str) else frozenset(configured or ()) + policy_names = frozenset(candidate.model_name for candidate in policy.candidates) + if not policy_names.issubset(configured_names): + missing = ", ".join(sorted(policy_names - configured_names)) + raise ValueError(f"auto_setup tier {tier_name} references models not present in tiers: {missing}") + return self + @model_validator(mode="after") def _validate_tier_param_placement(self) -> "ComplexityRouterConfig": """Reject a router setting written into a tier entry's request params. diff --git a/litellm/router_strategy/complexity_router/response_latency.py b/litellm/router_strategy/complexity_router/response_latency.py new file mode 100644 index 00000000000..c77f9b46c23 --- /dev/null +++ b/litellm/router_strategy/complexity_router/response_latency.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +import hashlib +import math +import statistics +from collections.abc import Mapping, Sequence +from datetime import datetime, timedelta +from types import MappingProxyType +from typing import Final, Protocol + +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +from litellm._logging import verbose_router_logger + +from .config import AutoSetupCandidate, AutoSetupObjective + +_CACHE_PREFIX: Final = "auto_setup_response_latency:v1" +_CACHE_TTL_SECONDS: Final = 7 * 24 * 60 * 60 +_MAX_SAMPLES_PER_MODEL: Final = 20 +_MAX_EXPECTED_OUTPUT_SAMPLES: Final = 100 +_MIN_SAMPLES_PER_MODEL: Final = 2 +_OBJECT_ADAPTER: Final = TypeAdapter(object) +_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[str, object]) + + +class ResponseLatencyCache(Protocol): + async def async_get_cache( + self, + key: str, + parent_otel_span: object | None = None, + local_only: bool = False, + **kwargs: object, # kwargs-ok: matches the shared cache interface's optional transport parameters + ) -> object: ... + + async def async_set_cache( + self, + key: str, + value: object, + local_only: bool = False, + **kwargs: object, # kwargs-ok: matches the shared cache interface's optional transport parameters + ) -> None: ... + + +class ResponseLatencySample(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + ttft_ms: float = Field(ge=0) + output_tokens_per_ms: float = Field(gt=0) + + +class ResponseLatencyState(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + samples_by_model: Mapping[str, tuple[ResponseLatencySample, ...]] = Field( + default_factory=lambda: MappingProxyType({}) + ) + visible_output_tokens: tuple[int, ...] = () + + +def _cache_key(router_model_name: str, tier: str) -> str: + namespace: Final = hashlib.sha256(router_model_name.encode("utf-8")).hexdigest()[:24] + return f"{_CACHE_PREFIX}:{namespace}:{tier}" + + +def _state(value: object) -> ResponseLatencyState: + try: + return ResponseLatencyState.model_validate(value or MappingProxyType({})) + except ValidationError: + return ResponseLatencyState() + + +def _candidate_order( + candidates: Sequence[AutoSetupCandidate], + available_models: Sequence[str], + cold_start_model: str, +) -> tuple[str, ...]: + available: Final = frozenset(available_models) + ranked: Final = tuple(candidate.model_name for candidate in candidates if candidate.model_name in available) + extras: Final = tuple(model for model in available_models if model not in frozenset(ranked)) + ordered: Final = tuple(dict.fromkeys((*ranked, *extras))) + if cold_start_model not in ordered: + return ordered + return (cold_start_model, *(model for model in ordered if model != cold_start_model)) + + +def _median_prediction_ms(samples: Sequence[ResponseLatencySample], expected_output_tokens: float) -> float: + return statistics.median( + sample.ttft_ms + expected_output_tokens / sample.output_tokens_per_ms for sample in samples + ) + + +def _normalized_log(value: float, values: Sequence[float]) -> float: + logged: Final = tuple(math.log(max(item, 1e-12)) for item in values) + low: Final = min(logged) + high: Final = max(logged) + if low == high: + return 0.0 + return (math.log(max(value, 1e-12)) - low) / (high - low) + + +def _best_observed_model( + state: ResponseLatencyState, + candidates: Sequence[AutoSetupCandidate], + ordered_models: Sequence[str], + objective: AutoSetupObjective, +) -> str: + expected_tokens: Final = statistics.median(state.visible_output_tokens) if state.visible_output_tokens else 1.0 + latencies: Final[Mapping[str, float]] = MappingProxyType( + {model: _median_prediction_ms(state.samples_by_model[model], expected_tokens) for model in ordered_models} + ) + if objective == "task_completion_speed": + return min(ordered_models, key=lambda model: (latencies[model], ordered_models.index(model))) + + configured_costs: Final[Mapping[str, float]] = MappingProxyType( + {candidate.model_name: candidate.cost_per_completed_task_usd for candidate in candidates} + ) + known_costs: Final = tuple(configured_costs.values()) + conservative_unknown_cost: Final = max(known_costs) if known_costs else 1.0 + costs: Final[Mapping[str, float]] = MappingProxyType( + {model: configured_costs.get(model, conservative_unknown_cost) for model in ordered_models} + ) + latency_values: Final = tuple(latencies.values()) + cost_values: Final = tuple(costs.values()) + return min( + ordered_models, + key=lambda model: ( + math.hypot( + _normalized_log(latencies[model], latency_values), + _normalized_log(costs[model], cost_values), + ), + ordered_models.index(model), + ), + ) + + +async def select_runtime_response_model( + *, + router_cache: ResponseLatencyCache, + router_model_name: str, + tier: str, + candidates: Sequence[AutoSetupCandidate], + available_models: Sequence[str], + cold_start_model: str, + objective: AutoSetupObjective, +) -> str: + """Explore quality-admitted groups, then minimize equal-output response time.""" + + ordered: Final = _candidate_order(candidates, available_models, cold_start_model) + if not ordered: + raise ValueError(f"No Auto setup candidates remain for tier {tier}") + try: + cached = await router_cache.async_get_cache(key=_cache_key(router_model_name, tier)) + state: Final = _state(cached) + except Exception as exc: # noqa: BLE001 -- optional observations must never make a request unroutable + verbose_router_logger.warning("Auto setup response-latency cache read failed: %s", exc) + return ordered[0] + + counts: Final[Mapping[str, int]] = MappingProxyType( + {model: len(state.samples_by_model.get(model, ())) for model in ordered} + ) + least_observed: Final = min(counts.values()) + if least_observed < _MIN_SAMPLES_PER_MODEL: + return next(model for model in ordered if counts[model] == least_observed) + return _best_observed_model(state, candidates, ordered, objective) + + +def _seconds(value: object) -> float | None: + if isinstance(value, datetime): + return value.timestamp() + if isinstance(value, timedelta): + return value.total_seconds() + if isinstance(value, bool) or not isinstance(value, int | float): + return None + return float(value) + + +def _field(value: object, key: str) -> object | None: + if isinstance(value, Mapping): + try: + return _OBJECT_MAPPING_ADAPTER.validate_python(value).get(key) + except ValidationError: + return None + try: + return _OBJECT_ADAPTER.validate_python(object.__getattribute__(value, key)) + except AttributeError: + return None + + +def _visible_output_tokens(response_obj: object) -> int | None: + usage = _field(response_obj, "usage") + if usage is None: + return None + total = _field(usage, "completion_tokens") + if total is None: + total = _field(usage, "output_tokens") + if isinstance(total, bool) or not isinstance(total, int | float) or total <= 0: + return None + details = _field(usage, "completion_tokens_details") + if details is None: + details = _field(usage, "output_tokens_details") + reasoning = _field(details, "reasoning_tokens") if details is not None else None + reasoning_tokens = float(reasoning) if isinstance(reasoning, int | float) and not isinstance(reasoning, bool) else 0 + return max(1, round(float(total) - reasoning_tokens)) + + +def response_latency_sample( + kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object +) -> tuple[ResponseLatencySample, int] | None: + start: Final = _seconds(start_time) + end: Final = _seconds(end_time) + visible_tokens: Final = _visible_output_tokens(response_obj) + if start is None or end is None or end <= start or visible_tokens is None: + return None + total_ms: Final = (end - start) * 1000 + completion_start: Final = _seconds(kwargs.get("completion_start_time")) + ttft_ms = ( + (completion_start - start) * 1000 if completion_start is not None and start < completion_start < end else 0.0 + ) + generation_ms: Final = max(total_ms - ttft_ms, 1e-6) + return ResponseLatencySample( + ttft_ms=ttft_ms, + output_tokens_per_ms=visible_tokens / generation_ms, + ), visible_tokens + + +async def record_runtime_response_latency( + *, + router_cache: ResponseLatencyCache, + router_model_name: str, + tier: str, + routed_model: str, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, +) -> None: + measured: Final = response_latency_sample(kwargs, response_obj, start_time, end_time) + if measured is None: + return + sample, visible_tokens = measured + key: Final = _cache_key(router_model_name, tier) + state: Final = _state(await router_cache.async_get_cache(key=key)) + samples: Final = (*state.samples_by_model.get(routed_model, ()), sample)[-_MAX_SAMPLES_PER_MODEL:] + updated_state: Final = ResponseLatencyState( + samples_by_model=MappingProxyType({**state.samples_by_model, routed_model: samples}), + visible_output_tokens=(*state.visible_output_tokens, visible_tokens)[-_MAX_EXPECTED_OUTPUT_SAMPLES:], + ) + await router_cache.async_set_cache(key=key, value=updated_state.model_dump(mode="json"), ttl=_CACHE_TTL_SECONDS) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index adbd4486032..7405a3720d6 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -9,8 +9,11 @@ from typing import Final, Literal, TypeAlias from pydantic import BaseModel, Field, computed_field, field_validator, model_validator -from litellm.router_strategy.complexity_router.auto_setup import AutoSetupQualityLevel -from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig +from litellm.router_strategy.complexity_router.config import ( + AutoSetupObjective, + AutoSetupQualityLevel, + ComplexityRouterConfig, +) from litellm.types.utils import StandardLoggingRoutingDecision DEFAULT_ROUTING_TEST_ROUTER_NAME: Final[str] = "auto_router_routing_test" @@ -49,13 +52,14 @@ class AutoRouterExcludedModelGroup(BaseModel): """A caller-visible reason Auto setup did not use an available model group.""" model_group: str - reason: Literal["no_benchmark_match", "mixed_model_group"] + reason: Literal["no_benchmark_match", "mixed_model_group", "pricing_unavailable"] class AutoRouterRecommendationResponse(BaseModel): """An editable complexity-router config generated from the caller's usable model groups.""" quality_level: AutoSetupQualityLevel + optimize_for: AutoSetupObjective snapshot_id: str snapshot_generated_at: str available_model_group_count: int = Field(ge=0) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 34c47d3f201..74a682b8f15 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2938,6 +2938,16 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): savings_baseline_model: str savings_baseline_deployment_id: str tier_litellm_params: Mapping[str, object] # writable-ok: Pydantic warns on ReadOnly TypedDict fields + auto_setup_snapshot_id: str # writable-ok: Pydantic warns on ReadOnly TypedDict fields + auto_setup_quality_level: Literal[ # writable-ok: Pydantic rejects ReadOnly fields + "economy", "balanced", "high", "max" + ] + auto_setup_optimize_for: Literal[ # writable-ok: Pydantic rejects ReadOnly fields + "cost", "task_completion_speed", "balanced" + ] + auto_setup_selection_mode: Literal[ # writable-ok: Pydantic rejects ReadOnly fields + "snapshot_ranked", "runtime_response_latency" + ] # Fields whose values quote the caller's prompt. Dropped when an operator turns message @@ -2966,6 +2976,10 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( "savings_baseline_model", "savings_baseline_deployment_id", "tier_litellm_params", + "auto_setup_snapshot_id", + "auto_setup_quality_level", + "auto_setup_optimize_for", + "auto_setup_selection_mode", } ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 278fb50e1dd..ccaf4d5a0e9 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -136,6 +136,7 @@ async def test_recommendation_uses_caller_groups_and_returns_an_editable_complex response = await get_auto_router_recommendation( user_api_key_dict=ADMIN, quality_level="economy", + optimize_for="task_completion_speed", ) authorize.assert_awaited_once_with(user_api_key_dict=ADMIN, team_id=None) @@ -144,8 +145,11 @@ async def test_recommendation_uses_caller_groups_and_returns_an_editable_complex assert response.excluded_model_groups == () assert "unavailable-group" not in response.matched_model_groups assert response.complexity_router_config.classifier_type == "heuristic_v2" - assert all(len(models) == 1 for models in response.complexity_router_config.tiers.values()) - assert "auto_setup" not in response.complexity_router_config.model_dump(mode="json", exclude_none=True) + assert response.complexity_router_config.auto_setup is not None + assert response.complexity_router_config.auto_setup.optimize_for == "task_completion_speed" + assert response.complexity_router_config.auto_setup.tier_policies["SIMPLE"].selection_mode == ( + "runtime_response_latency" + ) def test_auto_setup_pricing_honors_custom_token_rates_and_rejects_non_token_charges() -> None: @@ -194,7 +198,7 @@ def test_auto_setup_pricing_resolves_an_explicit_base_model_before_a_deployment_ @pytest.mark.asyncio -async def test_recommendation_excludes_mixed_and_unmatched_but_keeps_unpriced_groups( +async def test_recommendation_excludes_mixed_unmatched_and_unpriced_groups( monkeypatch: pytest.MonkeyPatch, ) -> None: import litellm.proxy.management_endpoints.auto_router_endpoints as endpoint_module @@ -222,14 +226,15 @@ async def test_recommendation_excludes_mixed_and_unmatched_but_keeps_unpriced_gr response = await get_auto_router_recommendation( user_api_key_dict=ADMIN, quality_level="max", + optimize_for="cost", ) assert response.available_model_group_count == 4 - assert set(response.matched_model_groups) <= {"safe", "unpriced"} - assert response.matched_model_groups + assert response.matched_model_groups == ("safe",) assert {(item.model_group, item.reason) for item in response.excluded_model_groups} == { ("mixed", "mixed_model_group"), ("unknown", "no_benchmark_match"), + ("unpriced", "pricing_unavailable"), } diff --git a/tests/test_litellm/router_strategy/test_complexity_router_auto_setup.py b/tests/test_litellm/router_strategy/test_complexity_router_auto_setup.py index 64a2e46f09a..9a19a5fb12c 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router_auto_setup.py +++ b/tests/test_litellm/router_strategy/test_complexity_router_auto_setup.py @@ -1,21 +1,32 @@ import math import random -from unittest.mock import MagicMock, patch +from itertools import product +from unittest.mock import AsyncMock, MagicMock, patch import pytest from pydantic import ValidationError +from litellm import Router from litellm.router_strategy.complexity_router.auto_setup import ( AutoRouterSnapshot, AutoSetupDeployment, AutoSetupDeploymentPricing, - _quality_completion_cost, # pyright: ignore[reportPrivateUsage] -- regression test for deployment-aware setup ranking + _speed_completion_cost, # pyright: ignore[reportPrivateUsage] -- regression test for multi-call trajectory pricing analyze_auto_setup_inventory, build_auto_setup_config, load_auto_router_snapshot, ) from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter -from litellm.router_strategy.complexity_router.config import ComplexityTier +from litellm.router_strategy.complexity_router.config import ( + AutoSetupCandidate, + ComplexityRouterConfig, + ComplexityTier, +) +from litellm.router_strategy.complexity_router.response_latency import ( + record_runtime_response_latency, + response_latency_sample, + select_runtime_response_model, +) def _available_refs(*model_names: str) -> dict[str, tuple[AutoSetupDeployment, ...]]: @@ -51,38 +62,44 @@ def test_snapshot_is_verified_and_quality_gate_is_recomputed_for_available_model snapshot=snapshot, available_model_deployments=available, quality_level="economy", + optimize_for="cost", ) maximum = build_auto_setup_config( snapshot=snapshot, available_model_deployments=available, quality_level="max", + optimize_for="cost", ) nano_only = build_auto_setup_config( snapshot=snapshot, available_model_deployments=_available_refs("gpt-5.4-nano"), quality_level="max", + optimize_for="cost", ) - assert snapshot.artifact_sha256 == "e191b5f7225e4da7a178eb5fe8780bcea24fb0166be23e2069338c6fe5522cd5" - assert economy.tiers["SIMPLE"] == ["gpt-5.4-nano"] + assert snapshot.artifact_sha256 == "bab7488212d40d96c5e43f80893c5ca2961d0e25160cb1aebbef36a2422b8a2b" + assert economy.tiers["SIMPLE"] == ["gpt-5.4-nano", "gpt-5.6-sol"] assert maximum.tiers["SIMPLE"] == ["gpt-5.6-sol"] assert nano_only.tiers["SIMPLE"] == ["gpt-5.4-nano"] -def test_auto_setup_emits_a_plain_static_complexity_router_config() -> None: +def test_speed_only_changes_auto_generated_selection_policy() -> None: config = build_auto_setup_config( snapshot=load_auto_router_snapshot(), available_model_deployments=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"), quality_level="economy", + optimize_for="task_completion_speed", ) - payload = config.model_dump(mode="json", exclude_none=True) - assert payload["classifier_type"] == "heuristic_v2" - assert "auto_setup" not in payload - assert all(len(models) == 1 for models in config.tiers.values()) + assert config.auto_setup is not None + assert config.auto_setup.tier_policies["SIMPLE"].selection_mode == "runtime_response_latency" + assert config.auto_setup.tier_policies["MEDIUM"].selection_mode == "runtime_response_latency" + assert config.auto_setup.tier_policies["COMPLEX"].selection_mode == "snapshot_ranked" + assert config.auto_setup.tier_policies["REASONING"].selection_mode == "snapshot_ranked" + assert ComplexityRouterConfig(tiers=config.tiers).auto_setup is None -def test_every_snapshot_alias_builds_all_four_quality_setups_even_without_price() -> None: +def test_every_priced_snapshot_alias_builds_all_twelve_auto_setups() -> None: snapshot = load_auto_router_snapshot() aliases = tuple( alias @@ -101,17 +118,25 @@ def test_every_snapshot_alias_builds_all_four_quality_setups_even_without_price( ) for alias, deployment in priced.items(): - eligible, excluded = analyze_auto_setup_inventory(snapshot, {alias: (deployment,)}) - assert eligible == (alias,) - assert excluded == () - for quality_level in ("economy", "balanced", "high", "max"): + if deployment.pricing is None: + eligible, excluded = analyze_auto_setup_inventory(snapshot, {alias: (deployment,)}) + assert not eligible + assert [(item.model_group, item.reason) for item in excluded] == [(alias, "pricing_unavailable")] + continue + for quality_level, objective in product( + ("economy", "balanced", "high", "max"), + ("cost", "task_completion_speed", "balanced"), + ): config = build_auto_setup_config( snapshot=snapshot, available_model_deployments={alias: (deployment,)}, quality_level=quality_level, + optimize_for=objective, ) assert set(config.tiers) == {"SIMPLE", "MEDIUM", "COMPLEX", "REASONING"} assert all(models == [alias] for models in config.tiers.values()) + assert config.auto_setup is not None + assert all(policy.candidates[0].model_name == alias for policy in config.auto_setup.tier_policies.values()) def test_full_priced_inventory_builds_deterministically_for_every_choice() -> None: @@ -130,24 +155,33 @@ def test_full_priced_inventory_builds_deterministically_for_every_choice() -> No } assert len(deployments) == 279 - selected_tiers: dict[str, dict[str, set[str]]] = {} - for quality_level in ("economy", "balanced", "high", "max"): + cost_tiers: dict[str, dict[str, set[str]]] = {} + for quality_level, objective in product( + ("economy", "balanced", "high", "max"), + ("cost", "task_completion_speed", "balanced"), + ): config = build_auto_setup_config( snapshot=snapshot, available_model_deployments=deployments, quality_level=quality_level, + optimize_for=objective, ) repeated = build_auto_setup_config( snapshot=snapshot, available_model_deployments=dict(reversed(tuple(deployments.items()))), quality_level=quality_level, + optimize_for=objective, ) assert config.model_dump(mode="json") == repeated.model_dump(mode="json") - assert all(len(models) == 1 for models in config.tiers.values()) + assert all(models and len(models) == len(set(models)) for models in config.tiers.values()) assert all(set(models).issubset(deployments) for models in config.tiers.values()) - selected_tiers[quality_level] = {tier: set(models) for tier, models in config.tiers.items()} + if objective == "cost": + cost_tiers[quality_level] = {tier: set(models) for tier, models in config.tiers.items()} - assert set(selected_tiers) == {"economy", "balanced", "high", "max"} + for tier in ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"): + assert cost_tiers["max"][tier] <= cost_tiers["high"][tier] + assert cost_tiers["high"][tier] <= cost_tiers["balanced"][tier] + assert cost_tiers["balanced"][tier] <= cost_tiers["economy"][tier] def test_seeded_mixed_inventories_never_select_unknown_or_unsafe_groups() -> None: @@ -179,11 +213,15 @@ def test_seeded_mixed_inventories_never_select_unknown_or_unsafe_groups() -> Non for index in range(rng.randint(0, 10)) } ) - for quality_level in ("economy", "balanced", "high", "max"): + for quality_level, objective in product( + ("economy", "balanced", "high", "max"), + ("cost", "task_completion_speed", "balanced"), + ): config = build_auto_setup_config( snapshot=snapshot, available_model_deployments=inventory, quality_level=quality_level, + optimize_for=objective, ) selected = {model_name for models in config.tiers.values() for model_name in models} assert selected @@ -220,11 +258,12 @@ def test_mixed_or_partially_unmatched_model_groups_fail_closed() -> None: snapshot=snapshot, available_model_deployments=inventory, quality_level="economy", + optimize_for="cost", ) assert all(models == ["safe"] for models in config.tiers.values()) -def test_missing_cost_falls_back_to_quality_instead_of_blocking_setup() -> None: +def test_schema_valid_missing_cost_and_zero_completion_evidence_fail_closed() -> None: raw = load_auto_router_snapshot().model_dump(mode="json") quality_profiles = raw["models"]["gpt-5.4-nano-xhigh"]["quality_by_complexity"].values() for profile in quality_profiles: @@ -235,29 +274,8 @@ def test_missing_cost_falls_back_to_quality_instead_of_blocking_setup() -> None: snapshot = AutoRouterSnapshot.model_validate(raw) eligible, excluded = analyze_auto_setup_inventory(snapshot, _available_refs("gpt-5.4-nano")) - assert eligible == ("gpt-5.4-nano",) - assert excluded == () - generated = build_auto_setup_config( - snapshot=snapshot, - available_model_deployments=_available_refs("gpt-5.4-nano"), - quality_level="max", - ) - assert all(models == ["gpt-5.4-nano"] for models in generated.tiers.values()) - - -def test_multiple_unpriced_models_choose_the_highest_quality_inside_the_gate() -> None: - inventory = { - model_name: (AutoSetupDeployment(model_refs=(model_name,), pricing=None),) - for model_name in ("gpt-5.6-sol", "gpt-5.4-nano") - } - - generated = build_auto_setup_config( - snapshot=load_auto_router_snapshot(), - available_model_deployments=inventory, - quality_level="max", - ) - - assert generated.tiers["SIMPLE"] == ["gpt-5.6-sol"] + assert eligible == () + assert [(item.model_group, item.reason) for item in excluded] == [("gpt-5.4-nano", "pricing_unavailable")] def test_provider_repricing_and_multi_deployment_groups_are_conservative() -> None: @@ -270,23 +288,42 @@ def test_provider_repricing_and_multi_deployment_groups_are_conservative() -> No model_refs=("gpt-5.6-sol",), pricing=AutoSetupDeploymentPricing(input_cost_per_token=0.00001, output_cost_per_token=0.00002), ) - profile = snapshot.models["gpt-5.6-sol-max"].quality_by_complexity["trivial"] - cheap_cost = _quality_completion_cost(profile, (cheap,)) - expensive_cost = _quality_completion_cost(profile, (expensive,)) - mixed_cost = _quality_completion_cost(profile, (cheap, expensive)) - - assert cheap_cost is not None - assert expensive_cost is not None - assert mixed_cost is not None - assert math.isclose(expensive_cost, cheap_cost * 10) - assert math.isclose(mixed_cost, expensive_cost) - config = build_auto_setup_config( snapshot=snapshot, available_model_deployments={"cheap": (cheap,), "expensive": (expensive,), "mixed-price": (cheap, expensive)}, quality_level="economy", + optimize_for="cost", ) - assert config.tiers["SIMPLE"] == ["cheap"] + assert config.auto_setup is not None + candidates = config.auto_setup.tier_policies["SIMPLE"].candidates + by_name = {candidate.model_name: candidate.cost_per_completed_task_usd for candidate in candidates} + assert tuple(candidate.model_name for candidate in candidates) == ("cheap", "expensive", "mixed-price") + assert math.isclose(by_name["expensive"], by_name["cheap"] * 10) + assert math.isclose(by_name["mixed-price"], by_name["expensive"]) + + +def test_multi_call_trajectory_cost_does_not_apply_single_request_context_thresholds() -> None: + snapshot = load_auto_router_snapshot() + speed = snapshot.models["gpt-5.6-sol-max"].task_completion_speed_by_complexity["complex"][0] + pricing = AutoSetupDeploymentPricing( + input_cost_per_token=0.000001, + output_cost_per_token=0.000002, + cache_read_input_token_cost=0.0000001, + input_cost_per_token_above_200k_tokens=0.001, + output_cost_per_token_above_200k_tokens=0.002, + cache_read_input_token_cost_above_200k_tokens=0.0001, + ) + deployment = AutoSetupDeployment(model_refs=("gpt-5.6-sol",), pricing=pricing) + + actual = _speed_completion_cost(speed, (deployment,)) + expected_attempt_cost = ( + (speed.mean_uncached_input_tokens or 0) * pricing.input_cost_per_token + + (speed.mean_cache_read_input_tokens or 0) * (pricing.cache_read_input_token_cost or 0) + + (speed.mean_output_tokens or 0) * pricing.output_cost_per_token + ) + + assert actual is not None + assert math.isclose(actual, expected_attempt_cost / speed.success_probability) @pytest.mark.parametrize( @@ -315,27 +352,164 @@ def test_snapshot_semantic_validation_rejects_inconsistent_artifacts(mutation: s AutoRouterSnapshot.model_validate(raw) -def test_auto_setup_result_has_no_runtime_only_configuration() -> None: +def test_auto_setup_cannot_change_adaptive_or_custom_tier_routers() -> None: generated = build_auto_setup_config( snapshot=load_auto_router_snapshot(), available_model_deployments=_available_refs("gpt-5.6-sol"), quality_level="max", + optimize_for="cost", ) - payload = generated.model_dump(mode="json", exclude_none=True) + payload = generated.model_dump(mode="json") - assert "auto_setup" not in payload - assert payload["adaptive"] is False - assert payload["classifier_type"] == "heuristic_v2" + with pytest.raises(ValidationError, match="auto_setup and adaptive"): + ComplexityRouterConfig.model_validate({**payload, "adaptive": True}) + with pytest.raises(ValidationError, match="built-in four-tier"): + ComplexityRouterConfig.model_validate( + { + **payload, + "tier_definitions": [ + {"name": "fast", "description": "Fast"}, + {"name": "strong", "description": "Strong"}, + ], + "tiers": {"fast": ["gpt-5.6-sol"], "strong": ["gpt-5.6-sol"]}, + "fallback_tier": "fast", + "classifier_type": "llm", + "classifier_llm_config": {"model": "gpt-5.6-sol"}, + } + ) + + +class _Cache: + def __init__(self) -> None: + self.values: dict[str, object] = {} + + async def async_get_cache( + self, + key: str, + parent_otel_span: object | None = None, + local_only: bool = False, + **kwargs: object, + ) -> object: + return self.values.get(key) + + async def async_set_cache( + self, + key: str, + value: object, + local_only: bool = False, + **kwargs: object, + ) -> None: + self.values[key] = value + + +def _candidate(model_name: str, cost: float) -> AutoSetupCandidate: + return AutoSetupCandidate( + model_name=model_name, + benchmark_model_id=f"benchmark-{model_name}", + quality_lower_bound=0.95, + cost_per_completed_task_usd=cost, + ) @pytest.mark.asyncio -async def test_generated_single_model_tiers_use_the_existing_router_selection_path() -> None: +async def test_easy_task_speed_learns_equal_output_response_time() -> None: + cache = _Cache() + candidates = (_candidate("cheap-slow", 0.01), _candidate("fast", 0.04)) + + assert ( + await select_runtime_response_model( + router_cache=cache, + router_model_name="auto", + tier="SIMPLE", + candidates=candidates, + available_models=("cheap-slow", "fast"), + cold_start_model="cheap-slow", + objective="task_completion_speed", + ) + == "cheap-slow" + ) + + for model, end in (("cheap-slow", 0.30), ("cheap-slow", 0.30), ("fast", 0.15), ("fast", 0.15)): + await record_runtime_response_latency( + router_cache=cache, + router_model_name="auto", + tier="SIMPLE", + routed_model=model, + kwargs={"completion_start_time": 0.10}, + response_obj={ + "usage": { + "completion_tokens": 20, + "completion_tokens_details": {"reasoning_tokens": 10}, + } + }, + start_time=0.0, + end_time=end, + ) + + measured = response_latency_sample( + {"completion_start_time": 0.10}, + {"usage": {"completion_tokens": 20, "completion_tokens_details": {"reasoning_tokens": 10}}}, + 0.0, + 0.30, + ) + assert measured is not None and measured[1] == 10 + assert ( + await select_runtime_response_model( + router_cache=cache, + router_model_name="auto", + tier="SIMPLE", + candidates=candidates, + available_models=("cheap-slow", "fast"), + cold_start_model="cheap-slow", + objective="task_completion_speed", + ) + == "fast" + ) + + +@pytest.mark.asyncio +async def test_easy_task_balanced_uses_both_live_response_time_and_expected_cost() -> None: + cache = _Cache() + candidates = ( + _candidate("fast-expensive", 1.0), + _candidate("middle", 0.02), + _candidate("cheap-slow", 0.01), + ) + for model, end in (("fast-expensive", 0.10), ("middle", 0.12), ("cheap-slow", 0.30)): + for _ in range(2): + await record_runtime_response_latency( + router_cache=cache, + router_model_name="auto", + tier="SIMPLE", + routed_model=model, + kwargs={}, + response_obj={"usage": {"completion_tokens": 10}}, + start_time=0.0, + end_time=end, + ) + + common = { + "router_cache": cache, + "router_model_name": "auto", + "tier": "SIMPLE", + "candidates": candidates, + "available_models": tuple(candidate.model_name for candidate in candidates), + "cold_start_model": "cheap-slow", + } + assert await select_runtime_response_model(**common, objective="task_completion_speed") == "fast-expensive" + assert await select_runtime_response_model(**common, objective="balanced") == "middle" + + +@pytest.mark.asyncio +async def test_snapshot_policy_is_deterministic_but_manual_pool_keeps_random_selection() -> None: generated = build_auto_setup_config( snapshot=load_auto_router_snapshot(), available_model_deployments=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"), quality_level="economy", + optimize_for="cost", ) router_instance = MagicMock() + router_instance.cache = AsyncMock() automatic = ComplexityRouter( model_name="auto", litellm_router_instance=router_instance, @@ -348,8 +522,71 @@ async def test_generated_single_model_tiers_use_the_existing_router_selection_pa ) assert await automatic._pick_model_for_tier(ComplexityTier.SIMPLE, None, None, {}) == "gpt-5.4-nano" + assert ( + await automatic._pick_model_for_tier( + ComplexityTier.SIMPLE, + None, + None, + {}, + allowed_models=("gpt-5.6-sol",), + ) + == "gpt-5.6-sol" + ) with patch( # test-quality-ok: pinning random choice is required to prove manual routers retain their existing selection path "litellm.router_strategy.complexity_router.complexity_router.random.choice", return_value="second" ) as pick: assert await manual._pick_model_for_tier(ComplexityTier.SIMPLE, None, None, {}) == "second" - pick.assert_called_once_with(["first", "second"]) + pick.assert_called_once_with(("first", "second")) + + +@pytest.mark.asyncio +async def test_live_latency_telemetry_is_recorded_only_for_an_auto_runtime_policy() -> None: + # test-quality-ok: this callback's observable contract is whether telemetry is emitted + generated = build_auto_setup_config( + snapshot=load_auto_router_snapshot(), + available_model_deployments=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"), + quality_level="economy", + optimize_for="task_completion_speed", + ) + router = Router( + model_list=[ + { + "model_name": "smart", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": generated.model_dump(mode="json"), + "complexity_router_default_model": "gpt-5.6-sol", + }, + }, + {"model_name": "gpt-5.6-sol", "litellm_params": {"model": "gpt-5.6-sol"}}, + {"model_name": "gpt-5.4-nano", "litellm_params": {"model": "gpt-5.4-nano"}}, + ] + ) + metadata = { + "routing_decision": { + "router_model_name": "smart", + "tier": "SIMPLE", + "routed_model": "gpt-5.4-nano", + "auto_setup_selection_mode": "runtime_response_latency", + } + } + + with patch( # test-quality-ok: patching the telemetry boundary avoids mutating the shared runtime latency cache + "litellm.router_strategy.complexity_router.response_latency.record_runtime_response_latency", + new_callable=AsyncMock, + ) as recorder: + await router._record_auto_setup_response_latency( + kwargs={"litellm_params": {"metadata": metadata}}, + completion_response={"usage": {"completion_tokens": 1}}, + start_time=0.0, + end_time=0.1, + ) + metadata["routing_decision"]["auto_setup_selection_mode"] = "snapshot_ranked" + await router._record_auto_setup_response_latency( + kwargs={"litellm_params": {"metadata": metadata}}, + completion_response={"usage": {"completion_tokens": 1}}, + start_time=0.0, + end_time=0.1, + ) + + recorder.assert_awaited_once() diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index e4236afc586..35f4f1c1a14 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -12703,7 +12703,7 @@ async def test_prompt_management_factory_marks_injection_for_every_deployment(mo assert bucket["litellm_gateway_injected_cache"] == "" -def test_get_configured_mode_reads_deployment_model_info(): +def test_get_configured_mode_reads_audio_speech_deployment_model_info(): router = Router( model_list=[ { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation.ts index 8a5b35e22ce..d13452ffd9a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation.ts @@ -1,7 +1,7 @@ import { useQuery } from "@tanstack/react-query"; import { getAutoRouterRecommendation, type AutoRouterRecommendationResponse } from "@/components/networking"; -import type { AutoSetupQualityLevel } from "@/components/add_model/build_complexity_router_config"; +import type { AutoSetupObjective, AutoSetupQualityLevel } from "@/components/add_model/build_complexity_router_config"; import { createQueryKeys } from "../common/queryKeysFactory"; const recommendationKeys = createQueryKeys("autoRouterRecommendation"); @@ -9,6 +9,7 @@ const recommendationKeys = createQueryKeys("autoRouterRecommendation"); interface UseAutoRouterRecommendationParams { accessToken: string; qualityLevel: AutoSetupQualityLevel; + optimizeFor: AutoSetupObjective; teamId?: string; enabled: boolean; } @@ -16,15 +17,16 @@ interface UseAutoRouterRecommendationParams { export const useAutoRouterRecommendation = ({ accessToken, qualityLevel, + optimizeFor, teamId, enabled, }: UseAutoRouterRecommendationParams) => { - const filters = { qualityLevel, ...(teamId ? { teamId } : {}) }; + const filters = { qualityLevel, optimizeFor, ...(teamId ? { teamId } : {}) }; const queryOptions = { queryKey: recommendationKeys.list({ filters, }), - queryFn: () => getAutoRouterRecommendation(accessToken, qualityLevel, teamId), + queryFn: () => getAutoRouterRecommendation(accessToken, qualityLevel, optimizeFor, teamId), enabled: enabled && Boolean(accessToken), staleTime: 30 * 1000, }; diff --git a/ui/litellm-dashboard/src/components/add_model/AutoSetupControls.tsx b/ui/litellm-dashboard/src/components/add_model/AutoSetupControls.tsx index e1252b2375b..d506c43af3a 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoSetupControls.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoSetupControls.tsx @@ -2,7 +2,13 @@ import type { AutoRouterRecommendationResponse } from "@/components/networking"; import { Button } from "@/components/ui/button"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; -import type { AutoSetupQualityLevel } from "./build_complexity_router_config"; +import type { AutoSetupObjective, AutoSetupQualityLevel } from "./build_complexity_router_config"; + +const OBJECTIVE_DESCRIPTION: Record = { + cost: "Spend less while staying within your selected quality level", + task_completion_speed: "Finish requests sooner, not just generate tokens faster", + balanced: "Balance expected cost with time to completion", +}; const QUALITY_OPTIONS = [ { value: "economy", label: "Economy" }, @@ -11,23 +17,26 @@ const QUALITY_OPTIONS = [ { value: "max", label: "Max" }, ]; -const QUALITY_DESCRIPTION: Record = { - economy: "Allows models up to 15 points below the smartest available option", - balanced: "Keeps models within 7 points of the smartest available option", - high: "Keeps models within 3 points of the smartest available option", - max: "Keeps models within 1 point of the smartest available option", -}; +const OBJECTIVE_OPTIONS = [ + { value: "cost", label: "Cost" }, + { value: "task_completion_speed", label: "Task completion speed" }, + { value: "balanced", label: "Balanced" }, +]; interface AutoSetupControlsProps { qualityLevel: AutoSetupQualityLevel; + optimizeFor: AutoSetupObjective; onQualityLevelChange: (value: AutoSetupQualityLevel) => void; + onOptimizeForChange: (value: AutoSetupObjective) => void; recommendation?: AutoRouterRecommendationResponse; recommendationEnabled: boolean; recommendationPending: boolean; recommendationFailed: boolean; recommendationError: unknown; requiresTeamScope: boolean; + hasAutoPolicy: boolean; onRetry: () => void; + onRegenerate: () => void; onBack: () => void; } @@ -40,16 +49,31 @@ const readyMessage = (recommendation: AutoRouterRecommendationResponse): string return `Uses ${count} of ${total} available model${total === 1 ? "" : "s"} across four complexity levels`; }; +const recommendationDescription = (optimizeFor: AutoSetupObjective): string => { + switch (optimizeFor) { + case "cost": + return "Favors the lowest expected cost while keeping each request within your selected quality level"; + case "task_completion_speed": + return "Easy requests use live response speed; difficult requests use benchmark completion time"; + case "balanced": + return "Balances expected cost with response speed and benchmark completion time"; + } +}; + export default function AutoSetupControls({ qualityLevel, + optimizeFor, onQualityLevelChange, + onOptimizeForChange, recommendation, recommendationEnabled, recommendationPending, recommendationFailed, recommendationError, requiresTeamScope, + hasAutoPolicy, onRetry, + onRegenerate, onBack, }: AutoSetupControlsProps) { const excludedCount = recommendation?.excluded_model_groups.length ?? 0; @@ -62,7 +86,7 @@ export default function AutoSetupControls({
Choose the outcome you want. LiteLLM will build the model tiers for you
- {recommendation && excludedCount > 0 && ( + {hasAutoPolicy && excludedCount > 0 && (
{excludedCount} model{excludedCount === 1 ? " was" : "s were"} left out because Auto setup could not compare {excludedCount === 1 ? "it" : "them"} safely @@ -73,7 +97,7 @@ export default function AutoSetupControls({ Back
-
+
- {QUALITY_DESCRIPTION[qualityLevel]}. Auto picks the lowest estimated completion cost inside that range, or - the smartest option when price is unavailable -
-
- Quality uses uncertainty-adjusted public benchmark scores for each complexity level + Higher levels keep each tier closer to the smartest available model
+
+ + +
{OBJECTIVE_DESCRIPTION[optimizeFor]}
+
{recommendationPending && recommendation === undefined && (
@@ -118,11 +156,19 @@ export default function AutoSetupControls({ )} {recommendation && (
-
Recommended setup
-
- {readyMessage(recommendation)}. Each tier is the lowest-cost model with comparable pricing that meets your - selected quality level +
+ {hasAutoPolicy ? "Recommended setup" : "Configuration edited"}
+
+ {hasAutoPolicy + ? `${readyMessage(recommendation)}. ${recommendationDescription(optimizeFor)}` + : "Your model changes are now treated as a manual configuration"} +
+ {!hasAutoPolicy && ( + + )}
)}
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 02cb0543e17..62e950c890c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -2,6 +2,7 @@ import { fireEvent, renderWithProviders, screen, within } from "../../../tests/t import userEvent from "@testing-library/user-event"; import { vi } from "vitest"; import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { detachAutoSetupAfterPoolEdit } from "./auto_setup_policy"; vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../../tests/mocks/complexityScorerDefaults"), @@ -44,6 +45,30 @@ const baseProps = { }; describe("ComplexityRouterConfig", () => { + it("keeps Auto policy for classifier edits and detaches it for model-pool edits", () => { + const automatic: ComplexityRouterConfigValue = { + ...defaultValue, + auto_setup: { + schema_version: "1", + snapshot_id: "snapshot-test", + snapshot_sha256: "a".repeat(64), + quality_level: "max", + optimize_for: "cost", + tier_policies: {}, + }, + }; + + expect(detachAutoSetupAfterPoolEdit(automatic, { ...automatic, classifier_type: "heuristic_v2" }).auto_setup).toBe( + automatic.auto_setup, + ); + expect( + detachAutoSetupAfterPoolEdit(automatic, { + ...automatic, + tiers: { ...automatic.tiers, SIMPLE: ["gpt-4"] }, + }).auto_setup, + ).toBeUndefined(); + }); + it("should render", () => { renderWithProviders(); expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 791fd1fcfbb..8e85e215619 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -48,6 +48,7 @@ import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; +import type { AutoSetupConfigPayload } from "./build_complexity_router_config"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; export type { CustomTierSet, TierRow } from "./tier_rows"; @@ -446,6 +447,8 @@ export interface ComplexityRouterConfigValue { * edit round-trip. */ tier_model_params?: TierModelParamsByTier; + /** Present only when the zero-setup Auto flow generated this editable config. */ + auto_setup?: AutoSetupConfigPayload; } /** Session affinity wins where a hand-authored config sets both, matching the backend's own `or`. */ diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 80be6ed6a81..7f31356c20b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -4,7 +4,7 @@ import { vi } from "vitest"; import AddAutoRouterTab from "./add_auto_router_tab"; import { toast } from "@/lib/toast"; import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; -import { getMissingTiersError } from "./build_complexity_router_config"; +import { getMissingTiersError, type AutoSetupConfigPayload } from "./build_complexity_router_config"; import { getSubmitBlockedReason } from "./add_auto_router_tab"; import { buildModelAvailability } from "@/lib/autorouter_presets"; import { testAutoRouterRouting } from "../networking"; @@ -169,17 +169,17 @@ describe("AddAutoRouterTab", () => { expect(useAutoRouterRecommendation).toHaveBeenCalledWith(expect.objectContaining({ enabled: false })); }); - it("shows the quality choice after Configure automatically is selected", async () => { + it("shows the two outcome choices after Configure automatically is selected", async () => { const user = userEvent.setup(); renderWithProviders(); await user.click(screen.getByTestId("configure-automatically-button")); expect(screen.getByTestId("auto-quality-selector")).toBeInTheDocument(); - expect(screen.queryByTestId("auto-objective-selector")).not.toBeInTheDocument(); + expect(screen.getByTestId("auto-objective-selector")).toBeInTheDocument(); expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument(); expect(useAutoRouterRecommendation).toHaveBeenCalledWith( - expect.objectContaining({ qualityLevel: "balanced", enabled: true }), + expect.objectContaining({ qualityLevel: "balanced", optimizeFor: "balanced", enabled: true }), ); }); @@ -195,12 +195,38 @@ describe("AddAutoRouterTab", () => { expect(screen.queryByTestId("auto-quality-selector")).not.toBeInTheDocument(); }); - it("submits the editable static config generated for the caller's available model groups", async () => { + it("submits the editable policy generated for the caller's available model groups", async () => { const user = userEvent.setup(); + const autoSetup: AutoSetupConfigPayload = { + schema_version: "1" as const, + snapshot_id: "snapshot-test", + snapshot_sha256: "a".repeat(64), + quality_level: "max" as const, + optimize_for: "task_completion_speed" as const, + tier_policies: Object.fromEntries( + ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"].map((tier) => [ + tier, + { + selection_mode: tier === "SIMPLE" || tier === "MEDIUM" ? "runtime_response_latency" : "snapshot_ranked", + candidates: [ + { + model_name: "smart-group", + benchmark_model_id: "benchmark-smart", + quality_lower_bound: 0.98, + cost_per_completed_task_usd: 0.04, + }, + ], + ...(tier === "SIMPLE" || tier === "MEDIUM" ? { cold_start_model: "smart-group" } : {}), + evidence_status: "test", + }, + ]), + ), + }; mockFetchAvailableModels.mockResolvedValue([{ model_group: "smart-group", mode: "chat" }]); const recommendationQuery = { data: { quality_level: "max", + optimize_for: "task_completion_speed", snapshot_id: "snapshot-test", snapshot_generated_at: "2026-09-03T00:00:00Z", available_model_group_count: 2, @@ -219,6 +245,7 @@ describe("AddAutoRouterTab", () => { deployment_affinity: false, modality_routing: false, modality_pin_override: false, + auto_setup: autoSetup, }, }, isPending: false, @@ -236,17 +263,9 @@ describe("AddAutoRouterTab", () => { await user.click(screen.getByRole("button", { name: /add auto router/i })); await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); - const submitted = vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config; - expect(submitted).toMatchObject({ - tiers: { - SIMPLE: ["smart-group"], - MEDIUM: ["smart-group"], - COMPLEX: ["smart-group"], - REASONING: ["smart-group"], - }, - classifier_type: "heuristic_v2", + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + auto_setup: autoSetup, }); - expect(submitted).not.toHaveProperty("auto_setup"); }); // Detailed Configuration starts collapsed after manual setup is selected; a caller opts into the diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index cf3dc7d2a58..b5d3703b0f7 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -30,11 +30,12 @@ import ComplexityRouterConfig, { DEFAULT_DEPLOYMENT_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, } from "./ComplexityRouterConfig"; -import { getRecommendationBlockedReason, type SetupMode } from "./auto_setup_policy"; +import { detachAutoSetupAfterPoolEdit, getRecommendationBlockedReason, type SetupMode } from "./auto_setup_policy"; import { KeywordTierRule } from "./KeywordTierRules"; import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching"; import { + AutoSetupObjective, AutoSetupQualityLevel, BuildComplexityRouterConfigParams, buildComplexityRouterConfig, @@ -187,6 +188,7 @@ const AddAutoRouterTab: React.FC = ({ const [modelAccessGroups, setModelAccessGroups] = useState([]); const [setupMode, setSetupMode] = useState("manual"); const [qualityLevel, setQualityLevel] = useState("balanced"); + const [optimizeFor, setOptimizeFor] = useState("balanced"); const appliedRecommendationKey = useRef(null); const [complexityRouterConfig, setComplexityRouterConfig] = useState({ @@ -278,6 +280,7 @@ const AddAutoRouterTab: React.FC = ({ const recommendationParams = { accessToken, qualityLevel, + optimizeFor, teamId: requiresTeamScope ? watchedTeamId : undefined, enabled: recommendationEnabled, }; @@ -354,6 +357,7 @@ const AddAutoRouterTab: React.FC = ({ const key = [ recommendation.snapshot_id, qualityLevel, + optimizeFor, watchedTeamId ?? "", ...recommendation.matched_model_groups, ].join(":"); @@ -362,7 +366,7 @@ const AddAutoRouterTab: React.FC = ({ setSelectedPreset(undefined); applyPrefill(buildPresetPrefill(recommendation.complexity_router_config, availability)); setDetailsExpanded(false); - }, [recommendation, setupMode, qualityLevel, watchedTeamId, availability, applyPrefill]); + }, [recommendation, setupMode, qualityLevel, optimizeFor, watchedTeamId, availability, applyPrefill]); const handlePresetChange = (presetKey: string | undefined) => { if (!presetKey || presetKey === "custom") { @@ -401,6 +405,11 @@ const AddAutoRouterTab: React.FC = ({ hasRecommendation: recommendation !== undefined, }; const recommendationBlockedReason = getRecommendationBlockedReason(recommendationStatus); + const regenerateAutoSetup = () => { + if (!recommendation) return; + applyPrefill(buildPresetPrefill(recommendation.complexity_router_config, availability)); + setDetailsExpanded(false); + }; const submitBlockedReason = recommendationBlockedReason ?? getSubmitBlockedReason( @@ -449,6 +458,7 @@ const AddAutoRouterTab: React.FC = ({ reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score, enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation, contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer, + autoSetup: complexityRouterConfig.auto_setup, }; const submitRecommendedRouter = async (name: string) => { @@ -588,14 +598,18 @@ const AddAutoRouterTab: React.FC = ({ {setupMode === "auto" ? ( void refetchRecommendation()} + onRegenerate={regenerateAutoSetup} onBack={returnToManualSetup} /> ) : ( @@ -700,7 +714,9 @@ const AddAutoRouterTab: React.FC = ({ onEditingTiersChange={setEditingTiers} modelInfo={modelInfo} value={complexityRouterConfig} - onChange={setComplexityRouterConfig} + onChange={(next) => + setComplexityRouterConfig((previous) => detachAutoSetupAfterPoolEdit(previous, next)) + } customTechnicalKeywords={customTechnicalKeywords} onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords} keywordTierRules={keywordTierRules} diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup_policy.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup_policy.ts index 08efd9ecad0..1ef3303d124 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup_policy.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup_policy.ts @@ -1,3 +1,5 @@ +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + export type SetupMode = "auto" | "manual"; export interface RecommendationStatus { @@ -23,3 +25,18 @@ export const getRecommendationBlockedReason = ({ } return null; }; + +const autoSetupPoolShape = (value: ComplexityRouterConfigValue): string => + JSON.stringify({ + tiers: value.tiers, + customTierSet: value.custom_tier_set, + tierModelParams: value.tier_model_params, + }); + +export const detachAutoSetupAfterPoolEdit = ( + previous: ComplexityRouterConfigValue, + next: ComplexityRouterConfigValue, +): ComplexityRouterConfigValue => + previous.auto_setup && autoSetupPoolShape(previous) !== autoSetupPoolShape(next) + ? { ...next, auto_setup: undefined } + : next; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 2a8454ae246..446dd2f7b61 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -138,6 +138,7 @@ export interface BuildComplexityRouterConfigParams { tierModelParams?: TierModelParamsByTier; enableContextWindowEscalation?: boolean; contextWindowEscalationBuffer?: number; + autoSetup?: AutoSetupConfigPayload; } /** @@ -196,9 +197,34 @@ export interface ComplexityRouterConfigPayload { enable_context_window_escalation?: boolean; context_window_escalation_buffer?: number; tier_model_configs?: Record; + auto_setup?: AutoSetupConfigPayload; } export type AutoSetupQualityLevel = "economy" | "balanced" | "high" | "max"; +export type AutoSetupObjective = "cost" | "task_completion_speed" | "balanced"; + +export interface AutoSetupCandidatePayload { + model_name: string; + benchmark_model_id: string; + quality_lower_bound: number; + cost_per_completed_task_usd: number; +} + +export interface AutoSetupTierPolicyPayload { + selection_mode: "snapshot_ranked" | "runtime_response_latency"; + candidates: AutoSetupCandidatePayload[]; + cold_start_model?: string | null; + evidence_status: string; +} + +export interface AutoSetupConfigPayload { + schema_version: "1"; + snapshot_id: string; + snapshot_sha256: string; + quality_level: AutoSetupQualityLevel; + optimize_for: AutoSetupObjective; + tier_policies: Record; +} export const serializeTierLabels = (tierLabels: ComplexityTierLabels | undefined): ComplexityTierLabels | undefined => { const renamed = TIER_KEYS.map((tier) => [tier, tierLabels?.[tier]?.trim() ?? ""] as const).filter( @@ -458,6 +484,7 @@ export const buildComplexityRouterConfig = ({ tierModelParams, enableContextWindowEscalation, contextWindowEscalationBuffer, + autoSetup, }: BuildComplexityRouterConfigParams): ComplexityRouterConfigPayload => { const serializedTierModelConfigs = customTierSet ? serializeTierModelConfigs( @@ -524,6 +551,7 @@ export const buildComplexityRouterConfig = ({ ...(contextWindowEscalationBuffer !== undefined && { context_window_escalation_buffer: contextWindowEscalationBuffer, }), + ...(autoSetup && { auto_setup: autoSetup }), ...scorerKnobs, }; if (!customTierSet) return payload; diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts index f24f3033901..8129d91de30 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts @@ -553,6 +553,7 @@ describe("managed keys survive an untouched open-and-save", () => { "fallback_tier", "classification_prompt", "hybrid_boundary_margin", + "auto_setup", ]); it("carries every managed key a built-in router can hold through hydrate then save", () => { @@ -565,6 +566,21 @@ describe("managed keys survive an untouched open-and-save", () => { expect(dropped).toEqual([]); }); + it("round-trips the stored Auto policy while its model pools stay unchanged", () => { + const autoSetup = { + schema_version: "1" as const, + snapshot_id: "snapshot-test", + snapshot_sha256: "a".repeat(64), + quality_level: "max" as const, + optimize_for: "cost" as const, + tier_policies: {}, + }; + const stored = { ...STORED, classifier_type: "heuristic" as const, auto_setup: autoSetup }; + const hydrated = hydrateComplexityRouterConfig(stored, undefined); + + expect(buildUpdatedComplexityRouterConfig(stored, hydrated).auto_setup).toEqual(autoSetup); + }); + it("drops a stored local-scorer threshold when the operator converts the router to custom tiers", () => { const hydrated = hydrateComplexityRouterConfig(STORED_ALL_MANAGED, undefined); const converted = { diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx index 54766df93ec..9d733d14d8b 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx @@ -38,6 +38,7 @@ import { hydratePlanModeMinTier, hydrateTierLabels, dryRunRejection, + type AutoSetupConfigPayload, } from "../add_model/build_complexity_router_config"; import { KeywordTierRule } from "../add_model/KeywordTierRules"; import { DEFAULT_MATCH_THRESHOLD } from "../add_model/SemanticKeywordMatching"; @@ -60,6 +61,7 @@ import ComplexityRouterConfig, { DEFAULT_DEPLOYMENT_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, } from "../add_model/ComplexityRouterConfig"; +import { detachAutoSetupAfterPoolEdit } from "../add_model/auto_setup_policy"; import { Dialog, DialogContent, @@ -114,6 +116,7 @@ export interface StoredComplexityRouterConfig { return_raw_model_name?: boolean; enable_context_window_escalation?: unknown; context_window_escalation_buffer?: unknown; + auto_setup?: AutoSetupConfigPayload; } /** @@ -202,6 +205,7 @@ export const hydrateComplexityRouterConfig = ( typeof parsedConfig.context_window_escalation_buffer === "number" ? parsedConfig.context_window_escalation_buffer : undefined, + auto_setup: parsedConfig.auto_setup, }; }; @@ -238,6 +242,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([ "reasoning_override_min_score", "enable_context_window_escalation", "context_window_escalation_buffer", + "auto_setup", ]); // Managed only when the caller passes the corresponding state. A caller that does not render @@ -343,6 +348,7 @@ export const buildUpdatedComplexityRouterConfig = ( tierModelParams: value.tier_model_params, enableContextWindowEscalation: value.enable_context_window_escalation, contextWindowEscalationBuffer: value.context_window_escalation_buffer, + autoSetup: value.auto_setup, }; const built = buildComplexityRouterConfig(builderParams); @@ -723,7 +729,7 @@ const EditAutoRouterModal: React.FC = ({ modelInfo={modelInfo} value={complexityRouterConfig} onChange={(config) => { - setComplexityRouterConfig(config); + setComplexityRouterConfig((previous) => detachAutoSetupAfterPoolEdit(previous, config)); }} customTechnicalKeywords={customTechnicalKeywords} onCustomTechnicalKeywordsChange={setCustomTechnicalKeywords} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index cb0f88b1d43..00ee0986c43 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -89,7 +89,11 @@ import type { CoordinationRedisTestResponse, } from "@/app/(dashboard)/caching/_components/coordination_redis_settings/types"; import { MCP_TOOLS_PREVIEW_FORBIDDEN_MESSAGE } from "./mcp_tools/constants"; -import type { AutoSetupQualityLevel, ComplexityRouterConfigPayload } from "./add_model/build_complexity_router_config"; +import type { + AutoSetupObjective, + AutoSetupQualityLevel, + ComplexityRouterConfigPayload, +} from "./add_model/build_complexity_router_config"; import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab"; import type { RoutingDecision } from "./view_logs/LogDetailsDrawer/RoutingDecisionCard"; @@ -2418,13 +2422,14 @@ export interface AutoRouterRoutingTestRequest { export interface AutoRouterRecommendationResponse { quality_level: AutoSetupQualityLevel; + optimize_for: AutoSetupObjective; snapshot_id: string; snapshot_generated_at: string; available_model_group_count: number; matched_model_groups: string[]; excluded_model_groups: Array<{ model_group: string; - reason: "no_benchmark_match" | "mixed_model_group"; + reason: "no_benchmark_match" | "mixed_model_group" | "pricing_unavailable"; }>; complexity_router_config: ComplexityRouterConfigPayload; } @@ -2432,12 +2437,14 @@ export interface AutoRouterRecommendationResponse { export const getAutoRouterRecommendation = async ( accessToken: string, qualityLevel: AutoSetupQualityLevel, + optimizeFor: AutoSetupObjective, teamId?: string, ): Promise => await apiClient.get("/auto_router/recommendation", { accessToken, query: { quality_level: qualityLevel, + optimize_for: optimizeFor, ...(teamId ? { team_id: teamId } : {}), }, }); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index ff482bd23b8..4f80abb19c0 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -294,6 +294,7 @@ export const buildPresetPrefill = ( return_raw_model_name: config.return_raw_model_name, enable_context_window_escalation: config.enable_context_window_escalation, context_window_escalation_buffer: config.context_window_escalation_buffer, + auto_setup: config.auto_setup, }, customTechnicalKeywords: config.custom_technical_keywords ?? [], keywordTierRules: hydrateKeywordTierRules(config.keyword_tier_rules ?? []), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 02dbfb98b5d..b9a181bde44 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23449,7 +23449,7 @@ export interface components { * Reason * @enum {string} */ - reason: "no_benchmark_match" | "mixed_model_group"; + reason: "no_benchmark_match" | "mixed_model_group" | "pricing_unavailable"; }; /** * AutoRouterPresetConfig @@ -23509,6 +23509,11 @@ export interface components { excluded_model_groups: components["schemas"]["AutoRouterExcludedModelGroup"][]; /** Matched Model Groups */ matched_model_groups: string[]; + /** + * Optimize For + * @enum {string} + */ + optimize_for: "cost" | "task_completion_speed" | "balanced"; /** * Quality Level * @enum {string} @@ -23592,6 +23597,67 @@ export interface components { /** @description The decision record this request would have written to its log row */ routing_decision: components["schemas"]["StandardLoggingRoutingDecision"]; }; + /** + * AutoSetupCandidate + * @description One benchmark-backed model group in an Auto-generated tier policy. + */ + AutoSetupCandidate: { + /** Benchmark Model Id */ + benchmark_model_id: string; + /** Cost Per Completed Task Usd */ + cost_per_completed_task_usd: number; + /** Model Name */ + model_name: string; + /** Quality Lower Bound */ + quality_lower_bound: number; + }; + /** + * AutoSetupConfig + * @description Provenance and selection policy stamped only by zero-setup Auto creation. + */ + AutoSetupConfig: { + /** + * Optimize For + * @enum {string} + */ + optimize_for: "cost" | "task_completion_speed" | "balanced"; + /** + * Quality Level + * @enum {string} + */ + quality_level: "economy" | "balanced" | "high" | "max"; + /** + * Schema Version + * @default 1 + * @constant + */ + schema_version: "1"; + /** Snapshot Id */ + snapshot_id: string; + /** Snapshot Sha256 */ + snapshot_sha256: string; + /** Tier Policies */ + tier_policies: { + [key: string]: components["schemas"]["AutoSetupTierPolicy"]; + }; + }; + /** + * AutoSetupTierPolicy + * @description How an Auto-generated tier chooses from its quality-admitted model groups. + */ + AutoSetupTierPolicy: { + /** Candidates */ + candidates: components["schemas"]["AutoSetupCandidate"][]; + /** Cold Start Model */ + cold_start_model?: string | null; + /** Evidence Status */ + evidence_status: string; + /** + * Selection Mode + * @enum {string} + */ + selection_mode: "snapshot_ranked" | "runtime_response_latency"; + }; /** BaseLitellmParams */ BaseLitellmParams: { /** @@ -34656,6 +34722,8 @@ export interface components { adaptive_eligible: "all" | "classified_tier"; /** @description Quality vs cost weights for adaptive selection (used when adaptive=True) */ adaptive_weights?: components["schemas"]["AdaptiveRouterWeights"]; + /** @description Benchmark snapshot and per-tier selection policy generated by the zero-setup Auto flow. Absent on manually configured routers, whose pool selection behavior is unchanged */ + auto_setup?: components["schemas"]["AutoSetupConfig"] | null; /** * Classification Mode * @description When to run the complexity classifier. 'every_request' (the default) classifies every inference request, including the tool-result continuation turns of an agentic loop. 'user_turn' classifies only requests whose newest turn is a new human ask and replays the session's held routing decision on continuation turns, which cuts classifier spend and eliminates mid-loop model switches. Continuations with no held decision to replay (no resolvable session_id, expired pin, fresh restart) still classify. Unlike session_affinity, a new human ask always re-classifies, so a session can still move tiers between asks. Suppressed when plugins are configured, for the same reason session_affinity is: a replayed decision would bypass the plugin pipeline. @@ -36012,6 +36080,23 @@ export interface components { * @description Per-request provenance for a pre-routing strategy (auto-router) decision. */ StandardLoggingRoutingDecision: { + /** + * Auto Setup Optimize For + * @enum {string} + */ + auto_setup_optimize_for?: "cost" | "task_completion_speed" | "balanced"; + /** + * Auto Setup Quality Level + * @enum {string} + */ + auto_setup_quality_level?: "economy" | "balanced" | "high" | "max"; + /** + * Auto Setup Selection Mode + * @enum {string} + */ + auto_setup_selection_mode?: "snapshot_ranked" | "runtime_response_latency"; + /** Auto Setup Snapshot Id */ + auto_setup_snapshot_id?: string; /** * Cause * @enum {string} @@ -41215,6 +41300,8 @@ export interface operations { query?: { /** @description How close every admitted model must be to the best available quality score */ quality_level?: "economy" | "balanced" | "high" | "max"; + /** @description What to minimize after the selected quality gate is applied */ + optimize_for?: "cost" | "task_completion_speed" | "balanced"; /** @description Team whose model access the recommendation must use */ team_id?: string | null; };