diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5d5a25e7cd6..689e9b5fec4 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -860,6 +860,7 @@ class LiteLLMRoutes(enum.Enum): # proxy admin, or team admin naming their own team via team_id "/auto_router/test_routing", "/auto_router/validate_complexity_router_config", + "/auto_router/recommendation", # Agent registry - reads are role-scoped and writes are proxy-admin-gated # inside agent_endpoints/endpoints.py *agent_management_routes, diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 21e652114bc..6b4ad77ec07 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -3,6 +3,7 @@ AUTO ROUTER MANAGEMENT ENDPOINTS POST /auto_router/test_routing - Route one request through an unsaved complexity-router config POST /auto_router/validate_complexity_router_config - Dry-run the complexity-router write gate without saving +GET /auto_router/recommendation - Build an editable config from the caller's available model groups """ from collections.abc import Mapping, Sequence @@ -10,7 +11,7 @@ from datetime import datetime, timedelta, timezone from itertools import chain, groupby from operator import attrgetter from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Final, Protocol +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, cast from uuid import uuid4 from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator @@ -52,6 +53,7 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterBenchmarkTotals, AutoRouterCacheBucket, AutoRouterCacheStats, + AutoRouterRecommendationResponse, AutoRouterRoutingTestRequest, AutoRouterRoutingTestResponse, ComplexityRouterConfigValidationRequest, @@ -334,6 +336,117 @@ async def validate_complexity_router_config( return ComplexityRouterConfigValidationResponse(valid=error is None, error=error) +def _available_model_refs( + llm_router: "Router", model_names: Sequence[str], team_id: str | None +) -> Mapping[str, tuple[str, ...]]: + refs: Final[dict[str, tuple[str, ...]]] = {} + for model_name in model_names: + deployments = llm_router.get_model_list(model_name=model_name, team_id=team_id) or () + resolved: list[str] = [model_name] + for raw_deployment in deployments: + deployment = cast(Mapping[str, object], raw_deployment) + litellm_params = deployment.get("litellm_params") + model_info = deployment.get("model_info") + if isinstance(litellm_params, Mapping): + typed_params = cast(Mapping[str, object], litellm_params) + resolved.extend( + value + for key in ("model", "base_model") + if isinstance(value := typed_params.get(key), str) and value + ) + if isinstance(model_info, Mapping): + base_model = cast(Mapping[str, object], model_info).get("base_model") + if isinstance(base_model, str) and base_model: + resolved.append(base_model) + refs[model_name] = tuple(dict.fromkeys(resolved)) + return MappingProxyType(refs) + + +@router.get( + "/auto_router/recommendation", + tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list + dependencies=[Depends(user_api_key_auth)], # mutable-ok: fastapi's decorator signature types dependencies as a list + response_model=AutoRouterRecommendationResponse, + status_code=status.HTTP_200_OK, +) +async def get_auto_router_recommendation( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + quality_level: Annotated[ + Literal["economy", "balanced", "high", "max"], + Query(description="How close every admitted model must be to the best available quality score"), + ] = "balanced", + optimize_for: Annotated[ + Literal["cost", "task_completion_speed", "balanced"], + Query(description="What to minimize after the selected quality gate is applied"), + ] = "balanced", + team_id: Annotated[str | None, Query(description="Team whose model access the recommendation must use")] = None, +) -> AutoRouterRecommendationResponse: + """Build an editable four-tier router from only the model groups this caller can use.""" + + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + user_model, + ) + from litellm.proxy.utils import get_available_models_for_user + from litellm.router_strategy.complexity_router.auto_setup import ( + build_auto_setup_config, + load_auto_router_snapshot, + ) + + await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=team_id) + if llm_router is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.no_llm_router.value}, # mutable-ok: HTTPException detail is JSON-shaped + ) + available_models: Final = await get_available_models_for_user( + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + general_settings=general_settings, + user_model=user_model, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + team_id=team_id, + user_api_key_cache=user_api_key_cache, + ) + snapshot: Final = load_auto_router_snapshot() + try: + config: Final = build_auto_setup_config( + snapshot=snapshot, + available_model_refs=_available_model_refs(llm_router, available_models, team_id), + quality_level=quality_level, + optimize_for=optimize_for, + ) + except ValueError as exc: + raise HTTPException( + status_code=400, + detail={ + "error": f"Could not build an Auto setup: {exc}" + }, # mutable-ok: HTTPException detail is JSON-shaped + ) from exc + matched: Final = tuple( + dict.fromkeys( + model + for tier_models in config.tiers.values() + for model in ((tier_models,) if isinstance(tier_models, str) else tier_models) + ) + ) + return AutoRouterRecommendationResponse( + quality_level=quality_level, + optimize_for=optimize_for, + snapshot_id=snapshot.snapshot_id, + snapshot_generated_at=snapshot.generated_at, + matched_model_groups=matched, + complexity_router_config=RequestComplexityRouterConfig.model_validate( + config.model_dump(mode="json", exclude_none=True) + ), + ) + + @router.post( "/auto_router/test_routing", tags=["model management"], # mutable-ok: fastapi's decorator signature types tags as a list diff --git a/litellm/router.py b/litellm/router.py index f33dfbba7bf..6a432b7d423 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7865,6 +7865,12 @@ class Router: # WS session wrappers fire with result=None; per-turn costs tracked by inner calls. if kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime"): return + await self._record_auto_setup_response_latency( + kwargs=kwargs, + completion_response=completion_response, + start_time=start_time, + end_time=end_time, + ) standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None) if standard_logging_object is None: raise ValueError("standard_logging_object is None") @@ -7976,6 +7982,55 @@ class Router: "litellm.router.Router::deployment_callback_on_success(): Exception occured - %s", e ) + async def _record_auto_setup_response_latency( + self, + *, + kwargs: Mapping[str, object], + completion_response: object, + start_time: object, + end_time: object, + ) -> None: + raw_metadata = cast(object, get_litellm_metadata_from_kwargs(dict(kwargs))) + if not isinstance(raw_metadata, Mapping): + return + metadata = cast(Mapping[str, object], raw_metadata) + raw_decision = metadata.get("routing_decision") + if not isinstance(raw_decision, Mapping): + return + decision = cast(Mapping[str, object], raw_decision) + if decision.get("auto_setup_selection_mode") != "runtime_response_latency": + return + router_model_name = decision.get("router_model_name") + tier = decision.get("tier") + routed_model = decision.get("routed_model") + if not isinstance(router_model_name, str) or not router_model_name: + return + if not isinstance(tier, str) or not tier: + return + if not isinstance(routed_model, str) or not routed_model: + return + registered: Final = self.complexity_routers.get(router_model_name) + if not registered or not any(tagged.strategy.config.auto_setup is not None for tagged in registered): + return + + from litellm.router_strategy.complexity_router.response_latency import ( + record_runtime_response_latency, + ) + + try: + await record_runtime_response_latency( + router_cache=self.cache, + router_model_name=router_model_name, + tier=tier, + routed_model=routed_model, + kwargs=kwargs, + response_obj=completion_response, + start_time=start_time, + end_time=end_time, + ) + except Exception as exc: # noqa: BLE001 -- optional telemetry must never fail a successful completion + verbose_router_logger.warning("Could not record Auto setup response latency: %s", exc) + def sync_deployment_callback_on_success( self, kwargs, # kwargs to completion diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index e3da70f50fe..d36ce9c2e9a 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -68,6 +68,47 @@ still resolve to a deployment in `model_list`; this configuration does not creat - abc ``` +### Zero-setup Auto configuration + +The dashboard's Add Router flow can generate the four tiers from the model groups the +current admin or team can actually use. Auto setup asks for two product choices: + +- Quality level: Economy, Balanced, High, or Max +- Optimize for: Cost, Task completion speed, or Balanced + +The quality level is a gate, not an optimization weight. For every complexity tier, +LiteLLM finds the best available model's conservative LiveBench quality score and admits +only models within the selected maximum regret: 15 percentage points for Economy, 7 for +Balanced, 3 for High, and 1 for Max. The objective then ranks only those admitted models. +This means Max still sends trivial work to a smaller model when that model is within one +point of the best available quality for trivial work. + +Cost is benchmark cost per completed task, not input or output token price. It is observed +request dollars divided by quality-adjusted completed questions in the same complexity +bucket, with partial benchmark credit counted as a fractional completion. + +Task completion speed uses two forms of evidence: + +- SIMPLE and MEDIUM tiers learn equal-output response time from this proxy's successful + requests: time to first token plus the time to generate the expected number of visible + output tokens. Reasoning tokens are excluded. The policy explores every quality-admitted + model twice before using a rolling seven-day sample +- COMPLEX and REASONING tiers use retry-adjusted Terminal-Bench wall-clock time to a + successful task. Results are compared only inside the same agent and benchmark cohort + and fall back to cost when comparable speed evidence is unavailable + +Balanced minimizes the combined log-normalized cost and completion-time scores after the +same quality gate. The recommendation stores its snapshot ID, objective, admitted models, +and rank order directly in the router configuration, so the generated result is visible +and editable. Changing a generated tier's model pool in the dashboard removes that Auto +selection policy and returns the tier to normal manual pool selection. + +These rules apply only when a router is created through Auto setup. Existing routers, +manual configurations, and LiteLLM's deployment-level latency routing are unchanged. The +bundled benchmark data is a versioned snapshot: creating or regenerating a router uses the +snapshot shipped with that LiteLLM version, while an already saved router keeps its stored +policy. + ### Heuristic v2 Set `classifier_type: heuristic_v2` to classify with the bundled calibrated diff --git a/litellm/router_strategy/complexity_router/artifacts/auto_router_snapshot_v0.json b/litellm/router_strategy/complexity_router/artifacts/auto_router_snapshot_v0.json new file mode 100644 index 00000000000..faecabd8cfe --- /dev/null +++ b/litellm/router_strategy/complexity_router/artifacts/auto_router_snapshot_v0.json @@ -0,0 +1,7042 @@ +{ + "activation_scope": { + "latency_objectives": [ + "task_completion_speed", + "balanced" + ], + "setup_mode": "auto" + }, + "artifact_sha256": "d0ea7e584dbb9ea37eb15ce20ae342612e0dfaa5801b553e71fc76818d7cef64", + "complexity_policy": { + "cumulative_shares": { + "complex": 1.0, + "simple": 0.5, + "standard": 0.75, + "trivial": 0.25 + }, + "method": "relative_quantiles", + "online_response_latency_formula": "time_to_first_token_ms + expected_visible_output_tokens / output_tokens_per_ms", + "speed_metric_by_complexity": { + "complex": "offline_task_completion_latency", + "simple": "online_response_latency", + "standard": "offline_task_completion_latency", + "trivial": "online_response_latency" + } + }, + "definitions": { + "cost": "Observed dollars divided by quality-adjusted completed questions in the same bucket.", + "online_response_latency": "Time to first token plus expected visible output generation time from LiteLLM's live observations.", + "quality": "Mean LiveBench subtask score within the inferred complexity bucket.", + "quality_lower_bound": "Mean subtask score minus 1.96 standard errors, floored at zero.", + "task_completion_speed": "Terminal-Bench expected wall-clock time to a successful task, including failed attempts." + }, + "generated_at": "2026-09-03T20:00:00Z", + "limitations": [ + "The snapshot is an offline prior; it does not replace live endpoint health or deployment latency routing.", + "Terminal-Bench duration is full agent-task wall time and is only compared inside the same agent/version cohort.", + "Trivial and simple requests require online response-latency evidence; long-horizon benchmark time is never used to rank them.", + "A reference speed route anchors on the cohort containing the highest-quality measured candidate; it never compares raw seconds across cohorts.", + "When no quality-admitted model has comparable speed evidence, speed and balanced routes explicitly fall back to cost.", + "LiveBench cost per completion treats partial benchmark credit as a fractional completion.", + "Reference routes use the full benchmark catalog; LiteLLM must recompute them after intersecting with the user's configured and capability-compatible models.", + "Benchmark variants include their measured reasoning effort; runtime mapping must not silently claim the score for a different effort setting." + ], + "models": { + "claude-fable-5-1-max-effort": { + "benchmark_model_id": "claude-fable-5-1-max-effort", + "identity": { + "benchmark_model_id": "claude-fable-5-1-max-effort", + "family_id": "claude-fable-5-1", + "is_routable": true, + "litellm_model_keys": [ + "anthropic.claude-fable-5-1", + "azure_ai/claude-fable-5-1", + "claude-fable-5-1", + "eu.anthropic.claude-fable-5-1", + "global.anthropic.claude-fable-5-1", + "us.anthropic.claude-fable-5-1", + "vertex_ai/claude-fable-5-1" + ], + "reasoning_effort": "max", + "required_parameters": { + "reasoning_effort": "max" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-fable-5-1-max-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 4.002891945931923, + "mean_request_cost_usd": 2.6602901785714286, + "quality_lower_bound": 0.6092130921684613, + "quality_score": 0.6661216666666666, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-fable-5-1-max-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.38505754057873887, + "mean_request_cost_usd": 0.34787109826589596, + "quality_lower_bound": 0.8425995986019852, + "quality_score": 0.9008116666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-fable-5-1-max-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 1.7827805688792642, + "mean_request_cost_usd": 1.5179469740634004, + "quality_lower_bound": 0.7415937065851377, + "quality_score": 0.8288983333333334, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-fable-5-1-max-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.11767196890092387, + "mean_request_cost_usd": 0.11522747875354107, + "quality_lower_bound": 0.9581461039814795, + "quality_score": 0.9790199999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-fable-5-1-max-effort", + "cohort_id": "claude-code@2.1.257", + "complexity": "complex", + "cost_per_completed_task_usd": 170.09694079545451, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 22.012545279411764, + "observed_duration_p50_ms": 2313576.159, + "observed_duration_p95_ms": 19944972.068, + "quality_lower_bound": 0.07382276936875576, + "retry_adjusted_completion_ms": 36630585.75418182, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.12941176470588237, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-fable-5-1" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-fable-5-1-max-effort", + "cohort_id": "claude-code@2.1.257", + "complexity": "simple", + "cost_per_completed_task_usd": 25.25749088095238, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 18.720257947058823, + "observed_duration_p50_ms": 2841125.8109999998, + "observed_duration_p95_ms": 9005283.476, + "quality_lower_bound": 0.6390752485008677, + "retry_adjusted_completion_ms": 5309329.376857143, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.7411764705882353, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-fable-5-1" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-fable-5-1-max-effort", + "cohort_id": "claude-code@2.1.257", + "complexity": "standard", + "cost_per_completed_task_usd": 27.663567212962963, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 18.672907868750002, + "observed_duration_p50_ms": 2673098.842, + "observed_duration_p95_ms": 7985724.741, + "quality_lower_bound": 0.5664032172496735, + "retry_adjusted_completion_ms": 4729345.715351852, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.675, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-fable-5-1" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-fable-5-1-max-effort", + "cohort_id": "claude-code@2.1.257", + "complexity": "trivial", + "cost_per_completed_task_usd": 20.434632119047624, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 16.092272793750002, + "observed_duration_p50_ms": 2008713.561, + "observed_duration_p95_ms": 11603729.906000001, + "quality_lower_bound": 0.6857763938182242, + "retry_adjusted_completion_ms": 4635716.655238096, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.7875, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-fable-5-1" + } + ] + } + }, + "claude-fable-5-max-effort": { + "benchmark_model_id": "claude-fable-5-max-effort", + "identity": { + "benchmark_model_id": "claude-fable-5-max-effort", + "family_id": "claude-fable-5", + "is_routable": true, + "litellm_model_keys": [ + "anthropic.claude-fable-5", + "azure_ai/claude-fable-5", + "claude-fable-5", + "deepinfra/anthropic/claude-fable-5", + "eu.anthropic.claude-fable-5", + "global.anthropic.claude-fable-5", + "us.anthropic.claude-fable-5", + "vertex_ai/claude-fable-5" + ], + "reasoning_effort": "max", + "required_parameters": { + "reasoning_effort": "max" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-fable-5-max-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 5.214221760086114, + "mean_request_cost_usd": 3.4582892857142857, + "quality_lower_bound": 0.5788725616068174, + "quality_score": 0.65567, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-fable-5-max-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.43912636553478873, + "mean_request_cost_usd": 0.399135838150289, + "quality_lower_bound": 0.8497278750259674, + "quality_score": 0.9057400000000001, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-fable-5-max-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 1.8625221673806187, + "mean_request_cost_usd": 1.580536311239193, + "quality_lower_bound": 0.7532045960680842, + "quality_score": 0.8290166666666666, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-fable-5-max-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.1611368906253183, + "mean_request_cost_usd": 0.15596345609065154, + "quality_lower_bound": 0.946566900924344, + "quality_score": 0.967098, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-fable-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "complex", + "cost_per_completed_task_usd": 392.82553275, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 23.107384279411765, + "observed_duration_p50_ms": 2042428.4970000002, + "observed_duration_p95_ms": 17828109.054, + "quality_lower_bound": 0.025384785707111474, + "retry_adjusted_completion_ms": 83197786.8262, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.058823529411764705, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-fable-5" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-fable-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "simple", + "cost_per_completed_task_usd": 33.590369561320756, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 20.94458337352941, + "observed_duration_p50_ms": 3535340.2729999996, + "observed_duration_p95_ms": 9474656.393, + "quality_lower_bound": 0.5172971118468915, + "retry_adjusted_completion_ms": 6346110.95354717, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.6235294117647059, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-fable-5" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-fable-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "standard", + "cost_per_completed_task_usd": 58.93160589516129, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 22.835997284375, + "observed_duration_p50_ms": 2997084.142, + "observed_duration_p95_ms": 10159133.293000001, + "quality_lower_bound": 0.2882439968827771, + "retry_adjusted_completion_ms": 9369974.166258065, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.3875, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-fable-5" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-fable-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "trivial", + "cost_per_completed_task_usd": 29.201946306034483, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 21.171411071875, + "observed_duration_p50_ms": 3460185.4905000003, + "observed_duration_p95_ms": 10187401.28, + "quality_lower_bound": 0.618557347158301, + "retry_adjusted_completion_ms": 5933162.700517242, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.725, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-fable-5" + } + ] + } + }, + "claude-opus-4-5-20251101-thinking-64k-high-effort": { + "benchmark_model_id": "claude-opus-4-5-20251101-thinking-64k-high-effort", + "identity": { + "benchmark_model_id": "claude-opus-4-5-20251101-thinking-64k-high-effort", + "family_id": "claude-opus-4-5-20251101", + "is_routable": true, + "litellm_model_keys": [ + "claude-opus-4-5-20251101" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-opus-4-5-20251101-thinking-64k-high-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 1.2955334819023985, + "mean_request_cost_usd": 0.6453375, + "quality_lower_bound": 0.34521477211231855, + "quality_score": 0.47735666666666665, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-opus-4-5-20251101-thinking-64k-high-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.3623535990181607, + "mean_request_cost_usd": 0.2938965317919075, + "quality_lower_bound": 0.769244886726077, + "quality_score": 0.8051866666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-opus-4-5-20251101-thinking-64k-high-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 0.9657541234098289, + "mean_request_cost_usd": 0.6872095100864554, + "quality_lower_bound": 0.646597808452617, + "quality_score": 0.7009949999999999, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-opus-4-5-20251101-thinking-64k-high-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.22691398212248595, + "mean_request_cost_usd": 0.22091416430594898, + "quality_lower_bound": 0.9585884810652954, + "quality_score": 0.9749399999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "claude-opus-4-6-thinking-auto-high-effort": { + "benchmark_model_id": "claude-opus-4-6-thinking-auto-high-effort", + "identity": { + "benchmark_model_id": "claude-opus-4-6-thinking-auto-high-effort", + "family_id": "claude-opus-4-6", + "is_routable": true, + "litellm_model_keys": [ + "azure_ai/claude-opus-4-6", + "claude-opus-4-6", + "perplexity/anthropic/claude-opus-4-6", + "vertex_ai/claude-opus-4-6" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-opus-4-6-thinking-auto-high-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 0.9723904214231778, + "mean_request_cost_usd": 0.5290098214285714, + "quality_lower_bound": 0.4365895928124619, + "quality_score": 0.5346133333333333, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-opus-4-6-thinking-auto-high-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.2008007702995054, + "mean_request_cost_usd": 0.17118612716763007, + "quality_lower_bound": 0.7932421041561762, + "quality_score": 0.8487183333333332, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-opus-4-6-thinking-auto-high-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 0.7138626149321894, + "mean_request_cost_usd": 0.49348991354466853, + "quality_lower_bound": 0.6303728077430275, + "quality_score": 0.6842783333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-opus-4-6-thinking-auto-high-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.09611682034141958, + "mean_request_cost_usd": 0.09330311614730878, + "quality_lower_bound": 0.9611363710759814, + "quality_score": 0.9749399999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "claude-opus-4-7-xhigh-effort": { + "benchmark_model_id": "claude-opus-4-7-xhigh-effort", + "identity": { + "benchmark_model_id": "claude-opus-4-7-xhigh-effort", + "family_id": "claude-opus-4-7", + "is_routable": true, + "litellm_model_keys": [ + "anthropic.claude-opus-4-7", + "au.anthropic.claude-opus-4-7", + "azure_ai/claude-opus-4-7", + "claude-opus-4-7", + "deepinfra/anthropic/claude-opus-4-7", + "eu.anthropic.claude-opus-4-7", + "global.anthropic.claude-opus-4-7", + "jp.anthropic.claude-opus-4-7", + "perplexity/anthropic/claude-opus-4-7", + "us.anthropic.claude-opus-4-7", + "vertex_ai/claude-opus-4-7" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-opus-4-7-xhigh-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 1.765312974843897, + "mean_request_cost_usd": 0.977340625, + "quality_lower_bound": 0.46192477930777903, + "quality_score": 0.5435150000000001, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-opus-4-7-xhigh-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.21927367996237826, + "mean_request_cost_usd": 0.18908179190751448, + "quality_lower_bound": 0.7956674213700154, + "quality_score": 0.8572016666666666, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-opus-4-7-xhigh-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 0.8121505178745887, + "mean_request_cost_usd": 0.6127403458213256, + "quality_lower_bound": 0.6611835790578724, + "quality_score": 0.7416183333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-opus-4-7-xhigh-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.047679003430232555, + "mean_request_cost_usd": 0.04648583569405099, + "quality_lower_bound": 0.947439528242222, + "quality_score": 0.9728220000000001, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "claude-opus-4-8-max-effort": { + "benchmark_model_id": "claude-opus-4-8-max-effort", + "identity": { + "benchmark_model_id": "claude-opus-4-8-max-effort", + "family_id": "claude-opus-4-8", + "is_routable": true, + "litellm_model_keys": [ + "anthropic.claude-opus-4-8", + "au.anthropic.claude-opus-4-8", + "azure_ai/claude-opus-4-8", + "bedrock/us-gov-east-1/anthropic.claude-opus-4-8", + "bedrock/us-gov-west-1/anthropic.claude-opus-4-8", + "claude-opus-4-8", + "deepinfra/anthropic/claude-opus-4-8", + "eu.anthropic.claude-opus-4-8", + "global.anthropic.claude-opus-4-8", + "jp.anthropic.claude-opus-4-8", + "us-gov.anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + "vertex_ai/claude-opus-4-8" + ], + "reasoning_effort": "max", + "required_parameters": { + "reasoning_effort": "max" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-opus-4-8-max-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 3.7899521877012035, + "mean_request_cost_usd": 2.087561160714286, + "quality_lower_bound": 0.4441463198800639, + "quality_score": 0.5437283333333333, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-opus-4-8-max-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.42221953473814827, + "mean_request_cost_usd": 0.36244335260115607, + "quality_lower_bound": 0.7881699970577629, + "quality_score": 0.8577533333333333, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-opus-4-8-max-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 1.1457071695113437, + "mean_request_cost_usd": 0.8780420749279538, + "quality_lower_bound": 0.6434622480125556, + "quality_score": 0.7433983333333334, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-opus-4-8-max-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.15555792693028503, + "mean_request_cost_usd": 0.151885552407932, + "quality_lower_bound": 0.9576762334934847, + "quality_score": 0.9749800000000001, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-opus-4-8-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "complex", + "cost_per_completed_task_usd": 658.821080875, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 15.501672491176471, + "observed_duration_p50_ms": 2628999.225, + "observed_duration_p95_ms": 17801145.046, + "quality_lower_bound": 0.006476375454219535, + "retry_adjusted_completion_ms": 210516238.57050002, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.023529411764705882, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-opus-4-8" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-opus-4-8-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "simple", + "cost_per_completed_task_usd": 80.75515165909091, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 20.901333370588237, + "observed_duration_p50_ms": 4935654.733, + "observed_duration_p95_ms": 10689830.749, + "quality_lower_bound": 0.1775797328865609, + "retry_adjusted_completion_ms": 20200870.892045453, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.25882352941176473, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-opus-4-8" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-opus-4-8-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "standard", + "cost_per_completed_task_usd": 193.26893900000002, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 16.9110321625, + "observed_duration_p50_ms": 3971969.2490000003, + "observed_duration_p95_ms": 8566031.918000001, + "quality_lower_bound": 0.043031454647439876, + "retry_adjusted_completion_ms": 48481955.61957143, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0875, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-opus-4-8" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-opus-4-8-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "trivial", + "cost_per_completed_task_usd": 43.279292521276595, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 25.42658435625, + "observed_duration_p50_ms": 6080651.175, + "observed_duration_p95_ms": 12414037.945, + "quality_lower_bound": 0.47803841126883023, + "retry_adjusted_completion_ms": 10616437.123489361, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.5875, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-opus-4-8" + } + ] + } + }, + "claude-opus-5-max-effort": { + "benchmark_model_id": "claude-opus-5-max-effort", + "identity": { + "benchmark_model_id": "claude-opus-5-max-effort", + "family_id": "claude-opus-5", + "is_routable": true, + "litellm_model_keys": [ + "anthropic.claude-opus-5", + "au.anthropic.claude-opus-5", + "azure_ai/claude-opus-5", + "claude-opus-5", + "deepinfra/anthropic/claude-opus-5", + "eu.anthropic.claude-opus-5", + "global.anthropic.claude-opus-5", + "jp.anthropic.claude-opus-5", + "openrouter/anthropic/claude-opus-5", + "us.anthropic.claude-opus-5", + "vertex_ai/claude-opus-5" + ], + "reasoning_effort": "max", + "required_parameters": { + "reasoning_effort": "max" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-opus-5-max-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 3.0495230322762903, + "mean_request_cost_usd": 1.9083642857142855, + "quality_lower_bound": 0.5271982605289206, + "quality_score": 0.6398033333333334, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-opus-5-max-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.11491162393410251, + "mean_request_cost_usd": 0.10078121387283237, + "quality_lower_bound": 0.8101252023432568, + "quality_score": 0.8775550000000001, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-opus-5-max-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 0.8529217089305349, + "mean_request_cost_usd": 0.6715219020172911, + "quality_lower_bound": 0.6461502379455998, + "quality_score": 0.7567566666666666, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-opus-5-max-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.04746918777279668, + "mean_request_cost_usd": 0.04608361581920904, + "quality_lower_bound": 0.947379644845522, + "quality_score": 0.973158, + "question_count": 354, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-opus-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "complex", + "cost_per_completed_task_usd": 281.3378472, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 16.549285129411764, + "observed_duration_p50_ms": 2207406.744, + "observed_duration_p95_ms": 17664645.654, + "quality_lower_bound": 0.025384785707111474, + "retry_adjusted_completion_ms": 94611233.12079999, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.058823529411764705, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-opus-5" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "claude-opus-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "simple", + "cost_per_completed_task_usd": 30.785845800847458, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 21.36899885, + "observed_duration_p50_ms": 3380587.579, + "observed_duration_p95_ms": 15045314.204, + "quality_lower_bound": 0.5895400497870746, + "retry_adjusted_completion_ms": 7657584.832338983, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.6941176470588235, + "task_count": 17, + "terminal_bench_model_id": "anthropic/claude-opus-5" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-opus-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "standard", + "cost_per_completed_task_usd": 35.05708062195122, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 17.96675381875, + "observed_duration_p50_ms": 3494617.0105, + "observed_duration_p95_ms": 7178773.611, + "quality_lower_bound": 0.4049314460370689, + "retry_adjusted_completion_ms": 7639667.404512195, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.5125, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-opus-5" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "claude-opus-5-max-effort", + "cohort_id": "claude-code@2.1.231", + "complexity": "trivial", + "cost_per_completed_task_usd": 20.582428286764706, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 17.49506404375, + "observed_duration_p50_ms": 2546369.6345, + "observed_duration_p95_ms": 10659490.976, + "quality_lower_bound": 0.7558657249889396, + "retry_adjusted_completion_ms": 5383495.158191176, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.85, + "task_count": 16, + "terminal_bench_model_id": "anthropic/claude-opus-5" + } + ] + } + }, + "claude-sonnet-4-6-thinking-auto-medium-effort": { + "benchmark_model_id": "claude-sonnet-4-6-thinking-auto-medium-effort", + "identity": { + "benchmark_model_id": "claude-sonnet-4-6-thinking-auto-medium-effort", + "family_id": "claude-sonnet-4-6", + "is_routable": true, + "litellm_model_keys": [ + "anthropic.claude-sonnet-4-6", + "au.anthropic.claude-sonnet-4-6", + "azure_ai/claude-sonnet-4-6", + "claude-sonnet-4-6", + "deepinfra/anthropic/claude-sonnet-4-6", + "eu.anthropic.claude-sonnet-4-6", + "global.anthropic.claude-sonnet-4-6", + "jp.anthropic.claude-sonnet-4-6", + "snowflake/claude-sonnet-4-6", + "us.anthropic.claude-sonnet-4-6", + "vertex_ai/claude-sonnet-4-6" + ], + "reasoning_effort": "medium", + "required_parameters": { + "reasoning_effort": "medium" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-sonnet-4-6-thinking-auto-medium-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 0.568415493233437, + "mean_request_cost_usd": 0.2852071428571429, + "quality_lower_bound": 0.37703186770250874, + "quality_score": 0.488105, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-sonnet-4-6-thinking-auto-medium-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.163557543653307, + "mean_request_cost_usd": 0.13361184971098267, + "quality_lower_bound": 0.738825421457911, + "quality_score": 0.8124633333333332, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-sonnet-4-6-thinking-auto-medium-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 0.5803005820722565, + "mean_request_cost_usd": 0.4407207492795389, + "quality_lower_bound": 0.6238595808835384, + "quality_score": 0.7377216666666667, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-sonnet-4-6-thinking-auto-medium-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.06498087858468846, + "mean_request_cost_usd": 0.059028895184135975, + "quality_lower_bound": 0.8446427881887083, + "quality_score": 0.9310579999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "claude-sonnet-5-xhigh-effort": { + "benchmark_model_id": "claude-sonnet-5-xhigh-effort", + "identity": { + "benchmark_model_id": "claude-sonnet-5-xhigh-effort", + "family_id": "claude-sonnet-5", + "is_routable": true, + "litellm_model_keys": [ + "anthropic.claude-sonnet-5", + "au.anthropic.claude-sonnet-5", + "azure_ai/claude-sonnet-5", + "bedrock/us-gov-east-1/anthropic.claude-sonnet-5", + "bedrock/us-gov-west-1/anthropic.claude-sonnet-5", + "claude-sonnet-5", + "deepinfra/anthropic/claude-sonnet-5", + "eu.anthropic.claude-sonnet-5", + "global.anthropic.claude-sonnet-5", + "jp.anthropic.claude-sonnet-5", + "us-gov.anthropic.claude-sonnet-5", + "us.anthropic.claude-sonnet-5", + "vertex_ai/claude-sonnet-5" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "claude-sonnet-5-xhigh-effort", + "complexity": "complex", + "cost_per_completed_task_usd": 1.535564958590551, + "mean_request_cost_usd": 0.8301674107142858, + "quality_lower_bound": 0.4851509363965742, + "quality_score": 0.5578083333333334, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "claude-sonnet-5-xhigh-effort", + "complexity": "simple", + "cost_per_completed_task_usd": 0.2661630950266414, + "mean_request_cost_usd": 0.22217456647398845, + "quality_lower_bound": 0.7827287342273122, + "quality_score": 0.828, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "claude-sonnet-5-xhigh-effort", + "complexity": "standard", + "cost_per_completed_task_usd": 0.7553995751446368, + "mean_request_cost_usd": 0.5736688760806916, + "quality_lower_bound": 0.6481641669625048, + "quality_score": 0.7392083333333334, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "claude-sonnet-5-xhigh-effort", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.07604438047164719, + "mean_request_cost_usd": 0.07383626062322945, + "quality_lower_bound": 0.955519401260248, + "quality_score": 0.973274, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "deepseek-v4-flash": { + "benchmark_model_id": "deepseek-v4-flash", + "identity": { + "benchmark_model_id": "deepseek-v4-flash", + "family_id": "deepseek-v4-flash", + "is_routable": true, + "litellm_model_keys": [ + "azure_ai/deepseek-v4-flash", + "dashscope/deepseek-v4-flash", + "deepinfra/deepseek-ai/DeepSeek-V4-Flash", + "deepseek-v4-flash", + "deepseek/deepseek-v4-flash", + "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash", + "fireworks_ai/deepseek-v4-flash", + "libertai/deepseek-v4-flash", + "novita/deepseek/deepseek-v4-flash", + "pinstripes/ps/deepseek-v4-flash", + "qwen_ai_platform/deepseek-v4-flash", + "qwencloud/deepseek-v4-flash", + "tencent/deepseek-v4-flash", + "tensormesh/deepseek-ai/DeepSeek-V4-Flash", + "wandb/deepseek-ai/DeepSeek-V4-Flash" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "deepseek-v4-flash", + "complexity": "complex", + "cost_per_completed_task_usd": 0.03171128009987993, + "mean_request_cost_usd": 0.01421919642857143, + "quality_lower_bound": 0.3364896272529753, + "quality_score": 0.43319166666666664, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "deepseek-v4-flash", + "complexity": "simple", + "cost_per_completed_task_usd": 0.013094683650279621, + "mean_request_cost_usd": 0.009395375722543352, + "quality_lower_bound": 0.603384170578614, + "quality_score": 0.70219, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "deepseek-v4-flash", + "complexity": "standard", + "cost_per_completed_task_usd": 0.02837889405353749, + "mean_request_cost_usd": 0.016063112391930836, + "quality_lower_bound": 0.5055896074804485, + "quality_score": 0.599025, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "deepseek-v4-flash", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.004025900568615041, + "mean_request_cost_usd": 0.0038747875354107645, + "quality_lower_bound": 0.9258556425341127, + "quality_score": 0.957196, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "deepseek-v4-flash-0731": { + "benchmark_model_id": "deepseek-v4-flash-0731", + "identity": { + "benchmark_model_id": "deepseek-v4-flash-0731", + "family_id": "deepseek-v4-flash-0731", + "is_routable": true, + "litellm_model_keys": [ + "azure_ai/DeepSeek-V4-Flash-0731", + "dashscope/deepseek-v4-flash-0731", + "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731", + "fireworks_ai/accounts/fireworks/models/deepseek-v4-flash-0731", + "fireworks_ai/deepseek-v4-flash-0731", + "novita/deepseek/deepseek-v4-flash-0731", + "perplexity/perplexity/deepseek-v4-flash-0731", + "qwen_ai_platform/deepseek-v4-flash-0731", + "qwencloud/deepseek-v4-flash-0731", + "scaleway/deepseek-v4-flash-0731", + "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731", + "wandb/deepseek-ai/DeepSeek-V4-Flash-0731" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "deepseek-v4-flash-0731", + "complexity": "complex", + "cost_per_completed_task_usd": 0.4394054395075869, + "mean_request_cost_usd": 0.22660624999999998, + "quality_lower_bound": 0.40376896581503474, + "quality_score": 0.509, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "deepseek-v4-flash-0731", + "complexity": "simple", + "cost_per_completed_task_usd": 0.0060579263531826775, + "mean_request_cost_usd": 0.005122832369942197, + "quality_lower_bound": 0.7696517274404475, + "quality_score": 0.84598, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "deepseek-v4-flash-0731", + "complexity": "standard", + "cost_per_completed_task_usd": 0.012699646412453419, + "mean_request_cost_usd": 0.008925936599423631, + "quality_lower_bound": 0.6365405113412037, + "quality_score": 0.7135733333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "deepseek-v4-flash-0731", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.001468206744000269, + "mean_request_cost_usd": 0.0014127478753541078, + "quality_lower_bound": 0.9283932717105589, + "quality_score": 0.960822, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "deepseek-v4-flash-vision-exp": { + "benchmark_model_id": "deepseek-v4-flash-vision-exp", + "identity": { + "benchmark_model_id": "deepseek-v4-flash-vision-exp", + "family_id": "deepseek-v4-flash-vision-exp", + "is_routable": true, + "litellm_model_keys": [ + "deepseek-v4-flash-vision-exp", + "deepseek/deepseek-v4-flash-vision-exp", + "novita/deepseek/deepseek-v4-flash-vision-exp" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "deepseek-v4-flash-vision-exp", + "complexity": "complex", + "cost_per_completed_task_usd": 0.1042448619785062, + "mean_request_cost_usd": 0.06216830357142857, + "quality_lower_bound": 0.5460797677435435, + "quality_score": 0.6141033333333333, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "deepseek-v4-flash-vision-exp", + "complexity": "simple", + "cost_per_completed_task_usd": 0.04363811907904243, + "mean_request_cost_usd": 0.03554161849710982, + "quality_lower_bound": 0.7162153297521393, + "quality_score": 0.813615, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "deepseek-v4-flash-vision-exp", + "complexity": "standard", + "cost_per_completed_task_usd": 0.06818042434558261, + "mean_request_cost_usd": 0.05142103746397694, + "quality_lower_bound": 0.6937983800511365, + "quality_score": 0.7546849999999999, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "deepseek-v4-flash-vision-exp", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.015498709656001796, + "mean_request_cost_usd": 0.014693767705382438, + "quality_lower_bound": 0.9176876013540679, + "quality_score": 0.9568619999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "deepseek-v4-pro": { + "benchmark_model_id": "deepseek-v4-pro", + "identity": { + "benchmark_model_id": "deepseek-v4-pro", + "family_id": "deepseek-v4-pro", + "is_routable": true, + "litellm_model_keys": [ + "azure_ai/deepseek-v4-pro", + "dashscope/deepseek-v4-pro", + "deepinfra/deepseek-ai/DeepSeek-V4-Pro", + "deepseek-v4-pro", + "deepseek/deepseek-v4-pro", + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro", + "fireworks_ai/deepseek-v4-pro", + "novita/deepseek/deepseek-v4-pro", + "openrouter/deepseek/deepseek-v4-pro", + "qwen_ai_platform/deepseek-v4-pro", + "qwencloud/deepseek-v4-pro", + "tencent/deepseek-v4-pro", + "together_ai/deepseek-ai/DeepSeek-V4-Pro", + "wandb/deepseek-ai/DeepSeek-V4-Pro" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "deepseek-v4-pro", + "complexity": "complex", + "cost_per_completed_task_usd": 0.12317544903160577, + "mean_request_cost_usd": 0.06060982142857143, + "quality_lower_bound": 0.38016323574171296, + "quality_score": 0.4806566666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "deepseek-v4-pro", + "complexity": "simple", + "cost_per_completed_task_usd": 0.037665943874090765, + "mean_request_cost_usd": 0.03045722543352601, + "quality_lower_bound": 0.7292064176433101, + "quality_score": 0.805635, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "deepseek-v4-pro", + "complexity": "standard", + "cost_per_completed_task_usd": 0.06925305608767511, + "mean_request_cost_usd": 0.048333717579250715, + "quality_lower_bound": 0.6217235810903863, + "quality_score": 0.6848133333333334, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "deepseek-v4-pro", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.012856284713601382, + "mean_request_cost_usd": 0.012492067988668556, + "quality_lower_bound": 0.9522556444359002, + "quality_score": 0.972156, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "deepseek-v4-pro-0813": { + "benchmark_model_id": "deepseek-v4-pro-0813", + "identity": { + "benchmark_model_id": "deepseek-v4-pro-0813", + "family_id": "deepseek-v4-pro-0813", + "is_routable": true, + "litellm_model_keys": [ + "deepinfra/deepseek-ai/DeepSeek-V4-Pro-0813", + "fireworks_ai/accounts/fireworks/models/deepseek-v4-pro-0813", + "novita/deepseek/deepseek-v4-pro-0813", + "openrouter/deepseek/deepseek-v4-pro-0813", + "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "deepseek-v4-pro-0813", + "complexity": "complex", + "cost_per_completed_task_usd": 0.08776797815427166, + "mean_request_cost_usd": 0.05000223214285714, + "quality_lower_bound": 0.4943714005386736, + "quality_score": 0.5668333333333333, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "deepseek-v4-pro-0813", + "complexity": "simple", + "cost_per_completed_task_usd": 0.03819422168482955, + "mean_request_cost_usd": 0.03241705202312139, + "quality_lower_bound": 0.7850763286351601, + "quality_score": 0.8482799999999999, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "deepseek-v4-pro-0813", + "complexity": "standard", + "cost_per_completed_task_usd": 0.06167011642368867, + "mean_request_cost_usd": 0.048700576368876085, + "quality_lower_bound": 0.6654041601132071, + "quality_score": 0.76577, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "deepseek-v4-pro-0813", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.012087533779340973, + "mean_request_cost_usd": 0.011813597733711049, + "quality_lower_bound": 0.9652508853504652, + "quality_score": 0.978196, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gemini-3.1-pro-preview-high": { + "benchmark_model_id": "gemini-3.1-pro-preview-high", + "identity": { + "benchmark_model_id": "gemini-3.1-pro-preview-high", + "family_id": "gemini-3.1-pro-preview", + "is_routable": true, + "litellm_model_keys": [ + "gemini-3.1-pro-preview", + "gemini/gemini-3.1-pro-preview", + "openrouter/google/gemini-3.1-pro-preview", + "vertex_ai/gemini-3.1-pro-preview" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gemini-3.1-pro-preview-high", + "complexity": "complex", + "cost_per_completed_task_usd": 0.9058612649184755, + "mean_request_cost_usd": 0.52495, + "quality_lower_bound": 0.40345610903956064, + "quality_score": 0.5499133333333334, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gemini-3.1-pro-preview-high", + "complexity": "simple", + "cost_per_completed_task_usd": 0.11967070803487916, + "mean_request_cost_usd": 0.09876069364161849, + "quality_lower_bound": 0.7734428742199247, + "quality_score": 0.8217166666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gemini-3.1-pro-preview-high", + "complexity": "standard", + "cost_per_completed_task_usd": 0.3724329493241841, + "mean_request_cost_usd": 0.2970204610951009, + "quality_lower_bound": 0.7542212833788828, + "quality_score": 0.8009816666666666, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gemini-3.1-pro-preview-high", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.0712105483066712, + "mean_request_cost_usd": 0.06959660056657223, + "quality_lower_bound": 0.9680781539505636, + "quality_score": 0.980234, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gemini-3.5-flash-high": { + "benchmark_model_id": "gemini-3.5-flash-high", + "identity": { + "benchmark_model_id": "gemini-3.5-flash-high", + "family_id": "gemini-3.5-flash", + "is_routable": true, + "litellm_model_keys": [ + "deepinfra/google/gemini-3.5-flash", + "gemini-3.5-flash", + "gemini/gemini-3.5-flash", + "vertex_ai/gemini-3.5-flash" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gemini-3.5-flash-high", + "complexity": "complex", + "cost_per_completed_task_usd": 0.716755285254874, + "mean_request_cost_usd": 0.4117705357142857, + "quality_lower_bound": 0.44704592118475794, + "quality_score": 0.5572033333333334, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gemini-3.5-flash-high", + "complexity": "simple", + "cost_per_completed_task_usd": 0.1168008244985926, + "mean_request_cost_usd": 0.096671387283237, + "quality_lower_bound": 0.773030678034419, + "quality_score": 0.8204416666666666, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gemini-3.5-flash-high", + "complexity": "standard", + "cost_per_completed_task_usd": 0.3734288238372025, + "mean_request_cost_usd": 0.262806916426513, + "quality_lower_bound": 0.6095857754421493, + "quality_score": 0.7054900000000001, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gemini-3.5-flash-high", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.055806475174513294, + "mean_request_cost_usd": 0.05406742209631728, + "quality_lower_bound": 0.9531187660114996, + "quality_score": 0.9703519999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gemini-3.5-flash-lite-high": { + "benchmark_model_id": "gemini-3.5-flash-lite-high", + "identity": { + "benchmark_model_id": "gemini-3.5-flash-lite-high", + "family_id": "gemini-3.5-flash-lite", + "is_routable": true, + "litellm_model_keys": [ + "gemini-3.5-flash-lite", + "gemini/gemini-3.5-flash-lite", + "vertex_ai/gemini-3.5-flash-lite" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gemini-3.5-flash-lite-high", + "complexity": "complex", + "cost_per_completed_task_usd": 0.24637079271564255, + "mean_request_cost_usd": 0.12193035714285715, + "quality_lower_bound": 0.3746970230436221, + "quality_score": 0.48975166666666664, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gemini-3.5-flash-lite-high", + "complexity": "simple", + "cost_per_completed_task_usd": 0.031234662803737268, + "mean_request_cost_usd": 0.020885838150289018, + "quality_lower_bound": 0.4962706700104881, + "quality_score": 0.6511566666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gemini-3.5-flash-lite-high", + "complexity": "standard", + "cost_per_completed_task_usd": 0.09241253386148097, + "mean_request_cost_usd": 0.04537579250720461, + "quality_lower_bound": 0.34552349534955934, + "quality_score": 0.526715, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gemini-3.5-flash-lite-high", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.016427271233884436, + "mean_request_cost_usd": 0.015356940509915014, + "quality_lower_bound": 0.9069738182941612, + "quality_score": 0.9343920000000001, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gemini-3.6-flash-high": { + "benchmark_model_id": "gemini-3.6-flash-high", + "identity": { + "benchmark_model_id": "gemini-3.6-flash-high", + "family_id": "gemini-3.6-flash", + "is_routable": true, + "litellm_model_keys": [ + "gemini-3.6-flash", + "gemini/gemini-3.6-flash", + "vertex_ai/gemini-3.6-flash" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gemini-3.6-flash-high", + "complexity": "complex", + "cost_per_completed_task_usd": 0.8991196289540533, + "mean_request_cost_usd": 0.4985566964285714, + "quality_lower_bound": 0.3831639512582842, + "quality_score": 0.5280516666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gemini-3.6-flash-high", + "complexity": "simple", + "cost_per_completed_task_usd": 0.09242984943163318, + "mean_request_cost_usd": 0.07708381502890173, + "quality_lower_bound": 0.7862437005368376, + "quality_score": 0.8313566666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gemini-3.6-flash-high", + "complexity": "standard", + "cost_per_completed_task_usd": 0.2864337111126976, + "mean_request_cost_usd": 0.19355389048991356, + "quality_lower_bound": 0.5777227110111925, + "quality_score": 0.6809850000000001, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gemini-3.6-flash-high", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.04192351604395637, + "mean_request_cost_usd": 0.04085467422096318, + "quality_lower_bound": 0.9592649709496708, + "quality_score": 0.9803139999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gemini-3.7-flash-high": { + "benchmark_model_id": "gemini-3.7-flash-high", + "identity": { + "benchmark_model_id": "gemini-3.7-flash-high", + "family_id": "gemini-3.7-flash", + "is_routable": true, + "litellm_model_keys": [ + "deepinfra/google/gemini-3.7-flash", + "gemini-3.7-flash", + "gemini/gemini-3.7-flash", + "vertex_ai/gemini-3.7-flash" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gemini-3.7-flash-high", + "complexity": "complex", + "cost_per_completed_task_usd": 0.5173201729804798, + "mean_request_cost_usd": 0.31210401785714287, + "quality_lower_bound": 0.5068762110807983, + "quality_score": 0.6021866666666666, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gemini-3.7-flash-high", + "complexity": "simple", + "cost_per_completed_task_usd": 0.05808227272675574, + "mean_request_cost_usd": 0.05018092485549133, + "quality_lower_bound": 0.8047468229043783, + "quality_score": 0.8612716666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gemini-3.7-flash-high", + "complexity": "standard", + "cost_per_completed_task_usd": 0.22283850186854895, + "mean_request_cost_usd": 0.17719020172910663, + "quality_lower_bound": 0.6998404208779329, + "quality_score": 0.779095, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gemini-3.7-flash-high", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.024636455346966592, + "mean_request_cost_usd": 0.02414787535410765, + "quality_lower_bound": 0.9698314889558245, + "quality_score": 0.984234, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gemini-3.7-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "complex", + "cost_per_completed_task_usd": 295.9275225750001, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.4815002655882368, + "observed_duration_p50_ms": 648803.339, + "observed_duration_p95_ms": 6388953.142, + "quality_lower_bound": 0.0020797335877093272, + "retry_adjusted_completion_ms": 167815171.641, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.011764705882352941, + "task_count": 17, + "terminal_bench_model_id": "gemini/gemini-3.7-flash" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gemini-3.7-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "simple", + "cost_per_completed_task_usd": 51.97473815625, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 2.445870030882353, + "observed_duration_p50_ms": 1034465.7279999999, + "observed_duration_p95_ms": 2791459.0560000003, + "quality_lower_bound": 0.018449962264952782, + "retry_adjusted_completion_ms": 25638198.509499997, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.047058823529411764, + "task_count": 17, + "terminal_bench_model_id": "gemini/gemini-3.7-flash" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gemini-3.7-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "standard", + "cost_per_completed_task_usd": 87.11567227500001, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 4.355783613750001, + "observed_duration_p50_ms": 1048099.63, + "observed_duration_p95_ms": 6347074.634, + "quality_lower_bound": 0.019613237127295572, + "retry_adjusted_completion_ms": 33926894.975499995, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.05, + "task_count": 16, + "terminal_bench_model_id": "gemini/gemini-3.7-flash" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gemini-3.7-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "trivial", + "cost_per_completed_task_usd": 14.627877795535715, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 5.1197572284375, + "observed_duration_p50_ms": 1198424.5915, + "observed_duration_p95_ms": 6527013.723, + "quality_lower_bound": 0.2545439440982969, + "retry_adjusted_completion_ms": 5402727.119178573, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.35, + "task_count": 16, + "terminal_bench_model_id": "gemini/gemini-3.7-flash" + } + ] + } + }, + "gemini-3.8-flash-high": { + "benchmark_model_id": "gemini-3.8-flash-high", + "identity": { + "benchmark_model_id": "gemini-3.8-flash-high", + "family_id": "gemini-3.8-flash", + "is_routable": true, + "litellm_model_keys": [ + "gemini-3.8-flash", + "gemini/gemini-3.8-flash", + "vertex_ai/gemini-3.8-flash" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gemini-3.8-flash-high", + "complexity": "complex", + "cost_per_completed_task_usd": 1.0221141700239293, + "mean_request_cost_usd": 0.6203821428571429, + "quality_lower_bound": 0.46610797022905415, + "quality_score": 0.59472, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gemini-3.8-flash-high", + "complexity": "simple", + "cost_per_completed_task_usd": 0.1581553011230906, + "mean_request_cost_usd": 0.13259768786127168, + "quality_lower_bound": 0.7479820892243612, + "quality_score": 0.8371833333333334, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gemini-3.8-flash-high", + "complexity": "standard", + "cost_per_completed_task_usd": 0.35172343321268257, + "mean_request_cost_usd": 0.2592948126801153, + "quality_lower_bound": 0.518237967153724, + "quality_score": 0.7214366666666666, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gemini-3.8-flash-high", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.06112004606148399, + "mean_request_cost_usd": 0.059734844192634556, + "quality_lower_bound": 0.9602156860944612, + "quality_score": 0.9803520000000001, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gemini-3.8-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "complex", + "cost_per_completed_task_usd": 0.0, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.0, + "observed_duration_p50_ms": 989905.341, + "observed_duration_p95_ms": 14753648.89, + "quality_lower_bound": 0.025384785707111474, + "retry_adjusted_completion_ms": 41792359.309, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.058823529411764705, + "task_count": 17, + "terminal_bench_model_id": "gemini/gemini-3.8-flash" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gemini-3.8-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "simple", + "cost_per_completed_task_usd": 0.0, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.0, + "observed_duration_p50_ms": 1238644.2179999999, + "observed_duration_p95_ms": 2836649.066, + "quality_lower_bound": 0.1193204211030341, + "retry_adjusted_completion_ms": 7225142.85575, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.18823529411764706, + "task_count": 17, + "terminal_bench_model_id": "gemini/gemini-3.8-flash" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gemini-3.8-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "standard", + "cost_per_completed_task_usd": 0.0, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.0, + "observed_duration_p50_ms": 1398407.2505, + "observed_duration_p95_ms": 5482899.748, + "quality_lower_bound": 0.02698897769798554, + "retry_adjusted_completion_ms": 28856834.1502, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0625, + "task_count": 16, + "terminal_bench_model_id": "gemini/gemini-3.8-flash" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gemini-3.8-flash-high", + "cohort_id": "mini-swe-agent@2.4.6", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.0, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.0, + "observed_duration_p50_ms": 1614114.02, + "observed_duration_p95_ms": 5601424.65, + "quality_lower_bound": 0.3574781238878701, + "retry_adjusted_completion_ms": 5122828.089918919, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.4625, + "task_count": 16, + "terminal_bench_model_id": "gemini/gemini-3.8-flash" + } + ] + } + }, + "glm-5.2": { + "benchmark_model_id": "glm-5.2", + "identity": { + "benchmark_model_id": "glm-5.2", + "family_id": "glm-5.2", + "is_routable": true, + "litellm_model_keys": [ + "cloudflare/@cf/zai-org/glm-5.2", + "dashscope/glm-5.2", + "deepinfra/zai-org/GLM-5.2", + "novita/zai-org/glm-5.2", + "perplexity/perplexity/glm-5.2", + "qwen_ai_platform/glm-5.2", + "qwencloud/glm-5.2", + "scaleway/glm-5.2", + "scx-ai/GLM-5.2", + "together_ai/zai-org/GLM-5.2", + "wandb/zai-org/GLM-5.2", + "zai/glm-5.2" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "glm-5.2", + "complexity": "complex", + "cost_per_completed_task_usd": 0.9582778958204534, + "mean_request_cost_usd": 0.505003125, + "quality_lower_bound": 0.4491256039182908, + "quality_score": 0.5292666666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "glm-5.2", + "complexity": "simple", + "cost_per_completed_task_usd": 0.09705821465623285, + "mean_request_cost_usd": 0.07582774566473989, + "quality_lower_bound": 0.7088720546635146, + "quality_score": 0.7714249999999999, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "glm-5.2", + "complexity": "standard", + "cost_per_completed_task_usd": 0.21812351935756738, + "mean_request_cost_usd": 0.15715561959654178, + "quality_lower_bound": 0.6503549332056257, + "quality_score": 0.7146266666666667, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "glm-5.2", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.044066457474518976, + "mean_request_cost_usd": 0.042318696883852695, + "quality_lower_bound": 0.9415584583178681, + "quality_score": 0.9563119999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "glm-5.3": { + "benchmark_model_id": "glm-5.3", + "identity": { + "benchmark_model_id": "glm-5.3", + "family_id": "glm-5.3", + "is_routable": true, + "litellm_model_keys": [ + "friendliai/zai-org/GLM-5.3", + "novita/zai-org/glm-5.3", + "together_ai/zai-org/GLM-5.3", + "zai/glm-5.3" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "glm-5.3", + "complexity": "complex", + "cost_per_completed_task_usd": 1.3760382986917519, + "mean_request_cost_usd": 0.7866937500000001, + "quality_lower_bound": 0.49798510274288055, + "quality_score": 0.585265, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "glm-5.3", + "complexity": "simple", + "cost_per_completed_task_usd": 0.2735818911125524, + "mean_request_cost_usd": 0.23069537572254334, + "quality_lower_bound": 0.8070017266219552, + "quality_score": 0.8443766666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "glm-5.3", + "complexity": "standard", + "cost_per_completed_task_usd": 0.6404002688409565, + "mean_request_cost_usd": 0.4496979827089338, + "quality_lower_bound": 0.6770506562429186, + "quality_score": 0.7044583333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "glm-5.3", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.0685497268325098, + "mean_request_cost_usd": 0.06634872521246458, + "quality_lower_bound": 0.9524140844470205, + "quality_score": 0.964978, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "glm-5.3-flash": { + "benchmark_model_id": "glm-5.3-flash", + "identity": { + "benchmark_model_id": "glm-5.3-flash", + "family_id": "glm-5.3-flash", + "is_routable": true, + "litellm_model_keys": [ + "friendliai/zai-org/GLM-5.3-Flash", + "together_ai/zai-org/GLM-5.3-Flash", + "zai/glm-5.3-flash" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "glm-5.3-flash", + "complexity": "complex", + "cost_per_completed_task_usd": 0.08444897731421191, + "mean_request_cost_usd": 0.04486071428571429, + "quality_lower_bound": 0.4821901387445875, + "quality_score": 0.5438916666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "glm-5.3-flash", + "complexity": "simple", + "cost_per_completed_task_usd": 0.012833186280785237, + "mean_request_cost_usd": 0.010308381502890174, + "quality_lower_bound": 0.7526212593040512, + "quality_score": 0.7971966666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "glm-5.3-flash", + "complexity": "standard", + "cost_per_completed_task_usd": 0.06498773223013737, + "mean_request_cost_usd": 0.03704236311239193, + "quality_lower_bound": 0.48108917724919115, + "quality_score": 0.5927183333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "glm-5.3-flash", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.0038063559086596067, + "mean_request_cost_usd": 0.0036212464589235127, + "quality_lower_bound": 0.937061284213534, + "quality_score": 0.951764, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gpt-5.2-2025-12-11-high": { + "benchmark_model_id": "gpt-5.2-2025-12-11-high", + "identity": { + "benchmark_model_id": "gpt-5.2-2025-12-11-high", + "family_id": "gpt-5.2-2025-12-11", + "is_routable": true, + "litellm_model_keys": [ + "azure/gpt-5.2-2025-12-11", + "gpt-5.2-2025-12-11" + ], + "reasoning_effort": "high", + "required_parameters": { + "reasoning_effort": "high" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.2-2025-12-11-high", + "complexity": "complex", + "cost_per_completed_task_usd": 0.9776107483996368, + "mean_request_cost_usd": 0.5017026785714286, + "quality_lower_bound": 0.4450192465475368, + "quality_score": 0.5147416666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.2-2025-12-11-high", + "complexity": "simple", + "cost_per_completed_task_usd": 0.10011894926348755, + "mean_request_cost_usd": 0.08136705202312139, + "quality_lower_bound": 0.7699878737605818, + "quality_score": 0.8092583333333333, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.2-2025-12-11-high", + "complexity": "standard", + "cost_per_completed_task_usd": 0.26096573855927013, + "mean_request_cost_usd": 0.20101815561959652, + "quality_lower_bound": 0.6435898983631713, + "quality_score": 0.7465216666666666, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.2-2025-12-11-high", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.032578981907034385, + "mean_request_cost_usd": 0.03160991501416431, + "quality_lower_bound": 0.9562658651509364, + "quality_score": 0.9741959999999998, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gpt-5.2-codex": { + "benchmark_model_id": "gpt-5.2-codex", + "identity": { + "benchmark_model_id": "gpt-5.2-codex", + "family_id": "gpt-5.2-codex", + "is_routable": true, + "litellm_model_keys": [ + "azure/gpt-5.2-codex", + "chatgpt/gpt-5.2-codex", + "gpt-5.2-codex", + "openrouter/openai/gpt-5.2-codex" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.2-codex", + "complexity": "complex", + "cost_per_completed_task_usd": 0.5030990089987945, + "mean_request_cost_usd": 0.25831205357142856, + "quality_lower_bound": 0.3796292105491461, + "quality_score": 0.5173, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.2-codex", + "complexity": "simple", + "cost_per_completed_task_usd": 0.1359313613809764, + "mean_request_cost_usd": 0.10803323699421966, + "quality_lower_bound": 0.7274478995993964, + "quality_score": 0.7884699999999999, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.2-codex", + "complexity": "standard", + "cost_per_completed_task_usd": 0.2566082051874501, + "mean_request_cost_usd": 0.18742420749279537, + "quality_lower_bound": 0.6556912811110588, + "quality_score": 0.72872, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.2-codex", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.046269902645509856, + "mean_request_cost_usd": 0.044631444759206794, + "quality_lower_bound": 0.9423117932813272, + "quality_score": 0.9641959999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gpt-5.4-mini-xhigh": { + "benchmark_model_id": "gpt-5.4-mini-xhigh", + "identity": { + "benchmark_model_id": "gpt-5.4-mini-xhigh", + "family_id": "gpt-5.4-mini", + "is_routable": true, + "litellm_model_keys": [ + "azure/gpt-5.4-mini", + "azure_ai/gpt-5.4-mini", + "gpt-5.4-mini" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.4-mini-xhigh", + "complexity": "complex", + "cost_per_completed_task_usd": 1.1086728650030395, + "mean_request_cost_usd": 0.5080651785714286, + "quality_lower_bound": 0.34943884542827364, + "quality_score": 0.45387833333333333, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.4-mini-xhigh", + "complexity": "simple", + "cost_per_completed_task_usd": 0.23784162886902593, + "mean_request_cost_usd": 0.1749434131736527, + "quality_lower_bound": 0.6573537305842576, + "quality_score": 0.7271216666666667, + "question_count": 334, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.4-mini-xhigh", + "complexity": "standard", + "cost_per_completed_task_usd": 0.47478333663016814, + "mean_request_cost_usd": 0.25883400576368876, + "quality_lower_bound": 0.4915611374028915, + "quality_score": 0.5752016666666667, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.4-mini-xhigh", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.05001269537263177, + "mean_request_cost_usd": 0.047810511363636364, + "quality_lower_bound": 0.913999334114345, + "quality_score": 0.9561959999999999, + "question_count": 352, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gpt-5.4-nano-xhigh": { + "benchmark_model_id": "gpt-5.4-nano-xhigh", + "identity": { + "benchmark_model_id": "gpt-5.4-nano-xhigh", + "family_id": "gpt-5.4-nano", + "is_routable": true, + "litellm_model_keys": [ + "azure/gpt-5.4-nano", + "azure_ai/gpt-5.4-nano", + "gpt-5.4-nano" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.4-nano-xhigh", + "complexity": "complex", + "cost_per_completed_task_usd": 0.4235539250688885, + "mean_request_cost_usd": 0.2074729357798165, + "quality_lower_bound": 0.3746452256531407, + "quality_score": 0.48795666666666665, + "question_count": 218, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.4-nano-xhigh", + "complexity": "simple", + "cost_per_completed_task_usd": 0.03996286705037176, + "mean_request_cost_usd": 0.030907738095238096, + "quality_lower_bound": 0.646704615489165, + "quality_score": 0.7700783333333333, + "question_count": 336, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.4-nano-xhigh", + "complexity": "standard", + "cost_per_completed_task_usd": 0.07862014091495734, + "mean_request_cost_usd": 0.055246176470588236, + "quality_lower_bound": 0.6036385132310647, + "quality_score": 0.6786633333333333, + "question_count": 340, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.4-nano-xhigh", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.013677653605205743, + "mean_request_cost_usd": 0.012910227272727271, + "quality_lower_bound": 0.8768336208515851, + "quality_score": 0.93505, + "question_count": 352, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gpt-5.4-xhigh": { + "benchmark_model_id": "gpt-5.4-xhigh", + "identity": { + "benchmark_model_id": "gpt-5.4-xhigh", + "family_id": "gpt-5.4", + "is_routable": true, + "litellm_model_keys": [ + "azure/eu/gpt-5.4", + "azure/gpt-5.4", + "azure/us/gpt-5.4", + "azure_ai/gpt-5.4", + "bedrock_mantle/openai.gpt-5.4", + "bedrock_mantle/us-gov-east-1/openai.gpt-5.4", + "bedrock_mantle/us-gov-west-1/openai.gpt-5.4", + "chatgpt/gpt-5.4", + "gpt-5.4" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.4-xhigh", + "complexity": "complex", + "cost_per_completed_task_usd": 1.2505701471408646, + "mean_request_cost_usd": 0.7417468749999999, + "quality_lower_bound": 0.49226355376413256, + "quality_score": 0.5819433333333334, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.4-xhigh", + "complexity": "simple", + "cost_per_completed_task_usd": 0.2511781656306905, + "mean_request_cost_usd": 0.21519682080924857, + "quality_lower_bound": 0.7908750479160342, + "quality_score": 0.858365, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.4-xhigh", + "complexity": "standard", + "cost_per_completed_task_usd": 0.4406923355369697, + "mean_request_cost_usd": 0.3466391930835735, + "quality_lower_bound": 0.6700680818124257, + "quality_score": 0.7634283333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.4-xhigh", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.06593190774567793, + "mean_request_cost_usd": 0.06425099150141643, + "quality_lower_bound": 0.9591866862403164, + "quality_score": 0.980236, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gpt-5.5-xhigh": { + "benchmark_model_id": "gpt-5.5-xhigh", + "identity": { + "benchmark_model_id": "gpt-5.5-xhigh", + "family_id": "gpt-5.5", + "is_routable": true, + "litellm_model_keys": [ + "azure/eu/gpt-5.5", + "azure/gpt-5.5", + "azure/us/gpt-5.5", + "azure_ai/gpt-5.5", + "bedrock_mantle/openai.gpt-5.5", + "gpt-5.5" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.5-xhigh", + "complexity": "complex", + "cost_per_completed_task_usd": 1.4552500894204172, + "mean_request_cost_usd": 0.8918142857142856, + "quality_lower_bound": 0.5116450006539032, + "quality_score": 0.5971316666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.5-xhigh", + "complexity": "simple", + "cost_per_completed_task_usd": 0.23943935618607748, + "mean_request_cost_usd": 0.21060982658959537, + "quality_lower_bound": 0.8255203002596379, + "quality_score": 0.8804416666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.5-xhigh", + "complexity": "standard", + "cost_per_completed_task_usd": 0.5132128101336504, + "mean_request_cost_usd": 0.4245420749279539, + "quality_lower_bound": 0.7103400163692298, + "quality_score": 0.8007049999999999, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.5-xhigh", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.0654670400159351, + "mean_request_cost_usd": 0.06398328611898017, + "quality_lower_bound": 0.9666846166236629, + "quality_score": 0.982156, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "gpt-5.6-luna-max": { + "benchmark_model_id": "gpt-5.6-luna-max", + "identity": { + "benchmark_model_id": "gpt-5.6-luna-max", + "family_id": "gpt-5.6-luna", + "is_routable": true, + "litellm_model_keys": [ + "azure/eu/gpt-5.6-luna", + "azure/gpt-5.6-luna", + "azure/us/gpt-5.6-luna", + "bedrock_mantle/openai.gpt-5.6-luna", + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna", + "global.openai.gpt-5.6-luna", + "gpt-5.6-luna", + "us.openai.gpt-5.6-luna" + ], + "reasoning_effort": "max", + "required_parameters": { + "reasoning_effort": "max" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.6-luna-max", + "complexity": "complex", + "cost_per_completed_task_usd": 0.5125596456280457, + "mean_request_cost_usd": 0.26013303571428575, + "quality_lower_bound": 0.4270514639974168, + "quality_score": 0.5046116666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.6-luna-max", + "complexity": "simple", + "cost_per_completed_task_usd": 0.11211393670540613, + "mean_request_cost_usd": 0.09202658959537573, + "quality_lower_bound": 0.7438523378026773, + "quality_score": 0.8184216666666666, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.6-luna-max", + "complexity": "standard", + "cost_per_completed_task_usd": 0.2442125605912569, + "mean_request_cost_usd": 0.17000951008645532, + "quality_lower_bound": 0.5904135803038684, + "quality_score": 0.6972016666666666, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.6-luna-max", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.026980681570241947, + "mean_request_cost_usd": 0.02592974504249292, + "quality_lower_bound": 0.9399069564413016, + "quality_score": 0.965314, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-luna-max", + "cohort_id": "codex@0.149.1", + "complexity": "complex", + "cost_per_completed_task_usd": null, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.6297226068235294, + "observed_duration_p50_ms": 812183.1919999999, + "observed_duration_p95_ms": 30386749.601, + "quality_lower_bound": 0.0, + "retry_adjusted_completion_ms": null, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-luna" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-luna-max", + "cohort_id": "codex@0.149.1", + "complexity": "simple", + "cost_per_completed_task_usd": 3.6673198505882363, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.7334639701176473, + "observed_duration_p50_ms": 1451894.867, + "observed_duration_p95_ms": 9231407.450000001, + "quality_lower_bound": 0.12878892241565565, + "retry_adjusted_completion_ms": 14908957.570411764, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.2, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-luna" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-luna-max", + "cohort_id": "codex@0.149.1", + "complexity": "standard", + "cost_per_completed_task_usd": 16.9274472, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 0.63477927, + "observed_duration_p50_ms": 1577440.2845, + "observed_duration_p95_ms": 5629013.643, + "quality_lower_bound": 0.012834336300788755, + "retry_adjusted_completion_ms": 54544322.523, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0375, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-luna" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-luna-max", + "cohort_id": "codex@0.149.1", + "complexity": "trivial", + "cost_per_completed_task_usd": 4.926968522203215, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 2.2787229415189874, + "observed_duration_p50_ms": 3137410.6835000003, + "observed_duration_p95_ms": 28950832.13, + "quality_lower_bound": 0.3574781238878701, + "retry_adjusted_completion_ms": 16030999.12881081, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.4625, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-luna" + } + ] + } + }, + "gpt-5.6-sol-max": { + "benchmark_model_id": "gpt-5.6-sol-max", + "identity": { + "benchmark_model_id": "gpt-5.6-sol-max", + "family_id": "gpt-5.6-sol", + "is_routable": true, + "litellm_model_keys": [ + "azure/eu/gpt-5.6-sol", + "azure/gpt-5.6-sol", + "azure/us/gpt-5.6-sol", + "bedrock_mantle/openai.gpt-5.6-sol", + "global.openai.gpt-5.6-sol", + "gpt-5.6-sol", + "us.openai.gpt-5.6-sol" + ], + "reasoning_effort": "max", + "required_parameters": { + "reasoning_effort": "max" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.6-sol-max", + "complexity": "complex", + "cost_per_completed_task_usd": 2.1494226631215954, + "mean_request_cost_usd": 1.347458482142857, + "quality_lower_bound": 0.5167214885397204, + "quality_score": 0.6125200000000001, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.6-sol-max", + "complexity": "simple", + "cost_per_completed_task_usd": 0.18762597508606915, + "mean_request_cost_usd": 0.16517803468208092, + "quality_lower_bound": 0.8273048746226402, + "quality_score": 0.8803166666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.6-sol-max", + "complexity": "standard", + "cost_per_completed_task_usd": 0.5463669630359826, + "mean_request_cost_usd": 0.4595057636887608, + "quality_lower_bound": 0.7208052023135284, + "quality_score": 0.815135, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.6-sol-max", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.03514393469872078, + "mean_request_cost_usd": 0.03444702549575071, + "quality_lower_bound": 0.9673442623054106, + "quality_score": 0.986196, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-sol-max", + "cohort_id": "codex@0.149.1", + "complexity": "complex", + "cost_per_completed_task_usd": 254.19713079999997, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 8.971663439999999, + "observed_duration_p50_ms": 1013924.23, + "observed_duration_p95_ms": 18501705.555, + "quality_lower_bound": 0.012074823021218326, + "retry_adjusted_completion_ms": 105534731.96500002, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.03529411764705882, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-sol" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-sol-max", + "cohort_id": "codex@0.149.1", + "complexity": "simple", + "cost_per_completed_task_usd": 14.184279841379311, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 4.839342534117647, + "observed_duration_p50_ms": 1373627.158, + "observed_duration_p95_ms": 3545131.2939999998, + "quality_lower_bound": 0.2492177176176207, + "retry_adjusted_completion_ms": 4732196.68475862, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.3411764705882353, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-sol" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-sol-max", + "cohort_id": "codex@0.149.1", + "complexity": "standard", + "cost_per_completed_task_usd": 21.92680297777778, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 7.400296005000001, + "observed_duration_p50_ms": 1280606.867, + "observed_duration_p95_ms": 7725407.675000001, + "quality_lower_bound": 0.24345459812221978, + "retry_adjusted_completion_ms": 5344111.560555556, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.3375, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-sol" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-sol-max", + "cohort_id": "codex@0.149.1", + "complexity": "trivial", + "cost_per_completed_task_usd": 13.002322996874996, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 10.401858397499998, + "observed_duration_p50_ms": 1556719.1935, + "observed_duration_p95_ms": 8823486.585, + "quality_lower_bound": 0.6995355220341885, + "retry_adjusted_completion_ms": 3411662.1276875, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.8, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-sol" + } + ] + } + }, + "gpt-5.6-terra-max": { + "benchmark_model_id": "gpt-5.6-terra-max", + "identity": { + "benchmark_model_id": "gpt-5.6-terra-max", + "family_id": "gpt-5.6-terra", + "is_routable": true, + "litellm_model_keys": [ + "azure/eu/gpt-5.6-terra", + "azure/gpt-5.6-terra", + "azure/us/gpt-5.6-terra", + "bedrock_mantle/openai.gpt-5.6-terra", + "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-terra", + "global.openai.gpt-5.6-terra", + "gpt-5.6-terra", + "us.openai.gpt-5.6-terra" + ], + "reasoning_effort": "max", + "required_parameters": { + "reasoning_effort": "max" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "gpt-5.6-terra-max", + "complexity": "complex", + "cost_per_completed_task_usd": 1.1240867780117232, + "mean_request_cost_usd": 0.6558124999999999, + "quality_lower_bound": 0.49669654030324234, + "quality_score": 0.5770166666666666, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "gpt-5.6-terra-max", + "complexity": "simple", + "cost_per_completed_task_usd": 0.1936266061380672, + "mean_request_cost_usd": 0.1675942196531792, + "quality_lower_bound": 0.8003444518177206, + "quality_score": 0.8676066666666666, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "gpt-5.6-terra-max", + "complexity": "standard", + "cost_per_completed_task_usd": 0.4724568735713442, + "mean_request_cost_usd": 0.3713230547550432, + "quality_lower_bound": 0.6492688041714549, + "quality_score": 0.759505, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "gpt-5.6-terra-max", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.04259387358760568, + "mean_request_cost_usd": 0.04132691218130312, + "quality_lower_bound": 0.9450457607873226, + "quality_score": 0.972196, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-terra-max", + "cohort_id": "codex@0.149.1", + "complexity": "complex", + "cost_per_completed_task_usd": 207.23259980000006, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 4.876061171764707, + "observed_duration_p50_ms": 760895.202, + "observed_duration_p95_ms": 17511504.763, + "quality_lower_bound": 0.006476375454219535, + "retry_adjusted_completion_ms": 143084489.17000002, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.023529411764705882, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-terra" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "gpt-5.6-terra-max", + "cohort_id": "codex@0.149.1", + "complexity": "simple", + "cost_per_completed_task_usd": 17.41710915542328, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.6883289976190476, + "observed_duration_p50_ms": 1243788.852, + "observed_duration_p95_ms": 3643298.026, + "quality_lower_bound": 0.13836125681504122, + "retry_adjusted_completion_ms": 7933672.004000001, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.21176470588235294, + "task_count": 17, + "terminal_bench_model_id": "openai/gpt-5.6-terra" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-terra-max", + "cohort_id": "codex@0.149.1", + "complexity": "standard", + "cost_per_completed_task_usd": 56.02047092, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.5012794325, + "observed_duration_p50_ms": 1329513.8054999998, + "observed_duration_p95_ms": 3397796.914, + "quality_lower_bound": 0.02698897769798554, + "retry_adjusted_completion_ms": 23351506.7788, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0625, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-terra" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "gpt-5.6-terra-max", + "cohort_id": "codex@0.149.1", + "complexity": "trivial", + "cost_per_completed_task_usd": 15.850715830434785, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 9.114161602500001, + "observed_duration_p50_ms": 1951635.4270000001, + "observed_duration_p95_ms": 14613717.367999999, + "quality_lower_bound": 0.465690919708387, + "retry_adjusted_completion_ms": 6175673.676978262, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.575, + "task_count": 16, + "terminal_bench_model_id": "openai/gpt-5.6-terra" + } + ] + } + }, + "grok-4.3": { + "benchmark_model_id": "grok-4.3", + "identity": { + "benchmark_model_id": "grok-4.3", + "family_id": "grok-4.3", + "is_routable": true, + "litellm_model_keys": [ + "azure_ai/grok-4.3", + "bedrock_mantle/us-gov-west-1/xai.grok-4.3", + "bedrock_mantle/xai.grok-4.3", + "xai/grok-4.3" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "grok-4.3", + "complexity": "complex", + "cost_per_completed_task_usd": 0.1859165423038312, + "mean_request_cost_usd": 0.07272321428571428, + "quality_lower_bound": 0.1928600724905164, + "quality_score": 0.33866, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "grok-4.3", + "complexity": "simple", + "cost_per_completed_task_usd": 0.04420760014121088, + "mean_request_cost_usd": 0.03145809248554913, + "quality_lower_bound": 0.6205897812643891, + "quality_score": 0.6990183333333334, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "grok-4.3", + "complexity": "standard", + "cost_per_completed_task_usd": 0.07347079856706272, + "mean_request_cost_usd": 0.04358991354466859, + "quality_lower_bound": 0.4608146638371801, + "quality_score": 0.5840483333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "grok-4.3", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.018345890446716372, + "mean_request_cost_usd": 0.01767025495750708, + "quality_lower_bound": 0.9484369021216132, + "quality_score": 0.964274, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "grok-4.5": { + "benchmark_model_id": "grok-4.5", + "identity": { + "benchmark_model_id": "grok-4.5", + "family_id": "grok-4.5", + "is_routable": true, + "litellm_model_keys": [ + "xai/grok-4.5" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "grok-4.5", + "complexity": "complex", + "cost_per_completed_task_usd": 0.44104901021713827, + "mean_request_cost_usd": 0.25584107142857143, + "quality_lower_bound": 0.46374943910594013, + "quality_score": 0.58254, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "grok-4.5", + "complexity": "simple", + "cost_per_completed_task_usd": 0.07142037040328908, + "mean_request_cost_usd": 0.05794421965317919, + "quality_lower_bound": 0.7272533331201472, + "quality_score": 0.8118366666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "grok-4.5", + "complexity": "standard", + "cost_per_completed_task_usd": 0.14598706555911886, + "mean_request_cost_usd": 0.10832363112391932, + "quality_lower_bound": 0.7117838299783479, + "quality_score": 0.73581, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "grok-4.5", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.03284308646047252, + "mean_request_cost_usd": 0.03237790368271955, + "quality_lower_bound": 0.9768668617803176, + "quality_score": 0.988118, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "grok-4.5", + "cohort_id": "grok-build@1.0.5", + "complexity": "complex", + "cost_per_completed_task_usd": null, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 2.71251968, + "observed_duration_p50_ms": 809437.804, + "observed_duration_p95_ms": 17566938.117999997, + "quality_lower_bound": 0.0, + "retry_adjusted_completion_ms": null, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.0, + "task_count": 17, + "terminal_bench_model_id": "xai/grok-4.5" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "grok-4.5", + "cohort_id": "grok-build@1.0.5", + "complexity": "simple", + "cost_per_completed_task_usd": 46.48075626666667, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 5.4683242666666665, + "observed_duration_p50_ms": 1772457.824, + "observed_duration_p95_ms": 5691062.943, + "quality_lower_bound": 0.06517287423683975, + "retry_adjusted_completion_ms": 22212528.7113, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.11764705882352941, + "task_count": 17, + "terminal_bench_model_id": "xai/grok-4.5" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "grok-4.5", + "cohort_id": "grok-build@1.0.5", + "complexity": "standard", + "cost_per_completed_task_usd": 130.93046981818182, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.2732617454545454, + "observed_duration_p50_ms": 1359035.736, + "observed_duration_p95_ms": 3994916.012, + "quality_lower_bound": 0.0068827488981252315, + "retry_adjusted_completion_ms": 62711641.6255, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.025, + "task_count": 16, + "terminal_bench_model_id": "xai/grok-4.5" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "grok-4.5", + "cohort_id": "grok-build@1.0.5", + "complexity": "trivial", + "cost_per_completed_task_usd": 43.48064595862069, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 15.76173416, + "observed_duration_p50_ms": 2102327.9115000004, + "observed_duration_p95_ms": 28832497.533, + "quality_lower_bound": 0.26570633724259446, + "retry_adjusted_completion_ms": 14499273.850896552, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.3625, + "task_count": 16, + "terminal_bench_model_id": "xai/grok-4.5" + } + ] + } + }, + "grok-4.6": { + "benchmark_model_id": "grok-4.6", + "identity": { + "benchmark_model_id": "grok-4.6", + "family_id": "grok-4.6", + "is_routable": true, + "litellm_model_keys": [ + "bedrock_mantle/xai.grok-4.6", + "global.xai.grok-4.6", + "us.xai.grok-4.6", + "xai/grok-4.6" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "grok-4.6", + "complexity": "complex", + "cost_per_completed_task_usd": 1.042706805778344, + "mean_request_cost_usd": 0.6088281249999999, + "quality_lower_bound": 0.47971095349363424, + "quality_score": 0.584845, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "grok-4.6", + "complexity": "simple", + "cost_per_completed_task_usd": 0.06858237773893723, + "mean_request_cost_usd": 0.05847023121387283, + "quality_lower_bound": 0.7905754736095916, + "quality_score": 0.8513383333333332, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "grok-4.6", + "complexity": "standard", + "cost_per_completed_task_usd": 0.14293474305698176, + "mean_request_cost_usd": 0.11195706051873199, + "quality_lower_bound": 0.7217334335863789, + "quality_score": 0.772135, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "grok-4.6", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.027443049307084175, + "mean_request_cost_usd": 0.02689886685552408, + "quality_lower_bound": 0.9709424951201708, + "quality_score": 0.9841179999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [ + { + "attempt_count": 85, + "benchmark_model_id": "grok-4.6", + "cohort_id": "grok-build@1.0.5", + "complexity": "complex", + "cost_per_completed_task_usd": 80.149236, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 3.7717287529411765, + "observed_duration_p50_ms": 1115803.779, + "observed_duration_p95_ms": 15627159.765, + "quality_lower_bound": 0.018449962264952782, + "retry_adjusted_completion_ms": 58126086.48225, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.047058823529411764, + "task_count": 17, + "terminal_bench_model_id": "xai/grok-4.6" + } + ], + "simple": [ + { + "attempt_count": 85, + "benchmark_model_id": "grok-4.6", + "cohort_id": "grok-build@1.0.5", + "complexity": "simple", + "cost_per_completed_task_usd": 39.875287432228916, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 7.505936457831325, + "observed_duration_p50_ms": 1821589.598, + "observed_duration_p95_ms": 4264177.431, + "quality_lower_bound": 0.1193204211030341, + "retry_adjusted_completion_ms": 10645207.7405625, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.18823529411764706, + "task_count": 17, + "terminal_bench_model_id": "xai/grok-4.6" + } + ], + "standard": [ + { + "attempt_count": 80, + "benchmark_model_id": "grok-4.6", + "cohort_id": "grok-build@1.0.5", + "complexity": "standard", + "cost_per_completed_task_usd": 472.3057235443037, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 11.807643088607593, + "observed_duration_p50_ms": 1404832.459, + "observed_duration_p95_ms": 7392303.36, + "quality_lower_bound": 0.0068827488981252315, + "retry_adjusted_completion_ms": 74351417.9385, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.025, + "task_count": 16, + "terminal_bench_model_id": "xai/grok-4.6" + } + ], + "trivial": [ + { + "attempt_count": 80, + "benchmark_model_id": "grok-4.6", + "cohort_id": "grok-build@1.0.5", + "complexity": "trivial", + "cost_per_completed_task_usd": 38.115191288888894, + "excluded_infrastructure_trials": 0, + "mean_attempt_cost_usd": 21.4397951, + "observed_duration_p50_ms": 1962060.6230000001, + "observed_duration_p95_ms": 9825563.117, + "quality_lower_bound": 0.45340937868656955, + "retry_adjusted_completion_ms": 4988289.024244444, + "source_id": "terminal-bench-4.0.0", + "success_probability": 0.5625, + "task_count": 16, + "terminal_bench_model_id": "xai/grok-4.6" + } + ] + } + }, + "grok-build-0.1": { + "benchmark_model_id": "grok-build-0.1", + "identity": { + "benchmark_model_id": "grok-build-0.1", + "family_id": "grok-build-0.1", + "is_routable": true, + "litellm_model_keys": [ + "xai/grok-build-0.1" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "grok-build-0.1", + "complexity": "complex", + "cost_per_completed_task_usd": 0.16789248462155656, + "mean_request_cost_usd": 0.08407366071428571, + "quality_lower_bound": 0.40526678365842894, + "quality_score": 0.49343000000000004, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "grok-build-0.1", + "complexity": "simple", + "cost_per_completed_task_usd": 0.0021711109969323223, + "mean_request_cost_usd": 0.0015864161849710983, + "quality_lower_bound": 0.6537908769552934, + "quality_score": 0.7247816666666668, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "grok-build-0.1", + "complexity": "standard", + "cost_per_completed_task_usd": 0.0034918518733508376, + "mean_request_cost_usd": 0.002343804034582133, + "quality_lower_bound": 0.6338637172183715, + "quality_score": 0.6733516666666667, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "grok-build-0.1", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.001605573981058883, + "mean_request_cost_usd": 0.0013872521246458924, + "quality_lower_bound": 0.762820104079826, + "quality_score": 0.886196, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "inkling-xhigh": { + "benchmark_model_id": "inkling-xhigh", + "identity": { + "benchmark_model_id": "inkling-xhigh", + "family_id": "inkling", + "is_routable": true, + "litellm_model_keys": [ + "deepinfra/thinkingmachines/Inkling", + "fireworks_ai/accounts/fireworks/models/inkling", + "together_ai/thinkingmachines/Inkling" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "inkling-xhigh", + "complexity": "complex", + "cost_per_completed_task_usd": 1.4951820821256014, + "mean_request_cost_usd": 0.7747223214285714, + "quality_lower_bound": 0.4138507462718552, + "quality_score": 0.5172866666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "inkling-xhigh", + "complexity": "simple", + "cost_per_completed_task_usd": 0.11300124240785248, + "mean_request_cost_usd": 0.08679479768786127, + "quality_lower_bound": 0.6879611199462832, + "quality_score": 0.7613733333333333, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "inkling-xhigh", + "complexity": "standard", + "cost_per_completed_task_usd": 0.2479303486614995, + "mean_request_cost_usd": 0.17703256484149854, + "quality_lower_bound": 0.687140938325973, + "quality_score": 0.7128933333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "inkling-xhigh", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.053020097447278515, + "mean_request_cost_usd": 0.05130538243626063, + "quality_lower_bound": 0.9304917509052512, + "quality_score": 0.9624520000000001, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "kimi-k2.6-thinking": { + "benchmark_model_id": "kimi-k2.6-thinking", + "identity": { + "benchmark_model_id": "kimi-k2.6-thinking", + "family_id": "kimi-k2.6-thinking", + "is_routable": false, + "litellm_model_keys": [], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "kimi-k2.6-thinking", + "complexity": "complex", + "cost_per_completed_task_usd": 0.5609214818286675, + "mean_request_cost_usd": 0.2897861607142857, + "quality_lower_bound": 0.4058833919145738, + "quality_score": 0.5106033333333334, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "kimi-k2.6-thinking", + "complexity": "simple", + "cost_per_completed_task_usd": 0.08119734639697186, + "mean_request_cost_usd": 0.06537341040462427, + "quality_lower_bound": 0.7560695680528867, + "quality_score": 0.7977683333333333, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "kimi-k2.6-thinking", + "complexity": "standard", + "cost_per_completed_task_usd": 0.22809804231052794, + "mean_request_cost_usd": 0.1387677233429395, + "quality_lower_bound": 0.5571036212027963, + "quality_score": 0.6188066666666666, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "kimi-k2.6-thinking", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.04589029570706512, + "mean_request_cost_usd": 0.04376685552407932, + "quality_lower_bound": 0.9187096389430004, + "quality_score": 0.9489000000000001, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "kimi-k2.7-code": { + "benchmark_model_id": "kimi-k2.7-code", + "identity": { + "benchmark_model_id": "kimi-k2.7-code", + "family_id": "kimi-k2.7-code", + "is_routable": true, + "litellm_model_keys": [ + "azure_ai/kimi-k2.7-code", + "cloudflare/@cf/moonshotai/kimi-k2.7-code", + "dashscope/kimi-k2.7-code", + "deepinfra/moonshotai/Kimi-K2.7-Code", + "moonshot/kimi-k2.7-code", + "novita/moonshotai/kimi-k2.7-code", + "perplexity/perplexity/kimi-k2.7-code", + "qwen_ai_platform/kimi-k2.7-code", + "qwencloud/kimi-k2.7-code", + "together_ai/moonshotai/Kimi-K2.7-Code", + "wandb/moonshotai/Kimi-K2.7-Code" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "kimi-k2.7-code", + "complexity": "complex", + "cost_per_completed_task_usd": 0.34182431702262395, + "mean_request_cost_usd": 0.16549955357142856, + "quality_lower_bound": 0.37480180324054907, + "quality_score": 0.48329, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "kimi-k2.7-code", + "complexity": "simple", + "cost_per_completed_task_usd": 0.04676665198251439, + "mean_request_cost_usd": 0.03778236994219653, + "quality_lower_bound": 0.7212085060481994, + "quality_score": 0.8057466666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "kimi-k2.7-code", + "complexity": "standard", + "cost_per_completed_task_usd": 0.16695183680534825, + "mean_request_cost_usd": 0.08871959654178674, + "quality_lower_bound": 0.44675198640057223, + "quality_score": 0.5578233333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "kimi-k2.7-code", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.018192812468927278, + "mean_request_cost_usd": 0.01726515580736544, + "quality_lower_bound": 0.9224390330931338, + "quality_score": 0.94655, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "kimi-k3": { + "benchmark_model_id": "kimi-k3", + "identity": { + "benchmark_model_id": "kimi-k3", + "family_id": "kimi-k3", + "is_routable": true, + "litellm_model_keys": [ + "deepinfra/moonshotai/Kimi-K3", + "fireworks_ai/accounts/fireworks/models/kimi-k3", + "fireworks_ai/kimi-k3", + "moonshot/kimi-k3", + "novita/moonshotai/kimi-k3", + "perplexity/perplexity/kimi-k3", + "together_ai/moonshotai/Kimi-K3" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "kimi-k3", + "complexity": "complex", + "cost_per_completed_task_usd": 1.124973962375416, + "mean_request_cost_usd": 0.69630625, + "quality_lower_bound": 0.5482513236653435, + "quality_score": 0.6231166666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "kimi-k3", + "complexity": "simple", + "cost_per_completed_task_usd": 0.1579575841136204, + "mean_request_cost_usd": 0.13721763005780346, + "quality_lower_bound": 0.8085757607273537, + "quality_score": 0.86872, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "kimi-k3", + "complexity": "standard", + "cost_per_completed_task_usd": 0.5183363932174508, + "mean_request_cost_usd": 0.36706023054755044, + "quality_lower_bound": 0.6428725660808584, + "quality_score": 0.73807, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "kimi-k3", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.05638490830737281, + "mean_request_cost_usd": 0.05494730878186969, + "quality_lower_bound": 0.9619821635145729, + "quality_score": 0.980274, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "minimax-m3": { + "benchmark_model_id": "minimax-m3", + "identity": { + "benchmark_model_id": "minimax-m3", + "family_id": "minimax-m3", + "is_routable": true, + "litellm_model_keys": [ + "deepinfra/MiniMaxAI/MiniMax-M3", + "fireworks_ai/accounts/fireworks/models/minimax-m3", + "fireworks_ai/minimax-m3", + "minimax/MiniMax-M3", + "novita/minimax/minimax-m3", + "tencent/minimax-m3", + "together_ai/MiniMaxAI/MiniMax-M3", + "wandb/MiniMaxAI/MiniMax-M3" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "minimax-m3", + "complexity": "complex", + "cost_per_completed_task_usd": 0.3130633806698212, + "mean_request_cost_usd": 0.14707723214285712, + "quality_lower_bound": 0.34031296465295013, + "quality_score": 0.4576766666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "minimax-m3", + "complexity": "simple", + "cost_per_completed_task_usd": 0.018624923352274145, + "mean_request_cost_usd": 0.013910693641618498, + "quality_lower_bound": 0.6614243653944405, + "quality_score": 0.7399033333333334, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "minimax-m3", + "complexity": "standard", + "cost_per_completed_task_usd": 0.04868696171064159, + "mean_request_cost_usd": 0.03023314121037464, + "quality_lower_bound": 0.5416649902386224, + "quality_score": 0.6333833333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "minimax-m3", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.00874328882093201, + "mean_request_cost_usd": 0.007756657223796033, + "quality_lower_bound": 0.8279299263482481, + "quality_score": 0.9090959999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "muse-spark-1.1-xhigh": { + "benchmark_model_id": "muse-spark-1.1-xhigh", + "identity": { + "benchmark_model_id": "muse-spark-1.1-xhigh", + "family_id": "muse-spark-1.1", + "is_routable": true, + "litellm_model_keys": [ + "meta/muse-spark-1.1" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "muse-spark-1.1-xhigh", + "complexity": "complex", + "cost_per_completed_task_usd": 1.0169169346234774, + "mean_request_cost_usd": 0.5882370535714285, + "quality_lower_bound": 0.48299424590997, + "quality_score": 0.5848166666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "muse-spark-1.1-xhigh", + "complexity": "simple", + "cost_per_completed_task_usd": 0.06071665890850331, + "mean_request_cost_usd": 0.04849884393063584, + "quality_lower_bound": 0.7024593150732495, + "quality_score": 0.7945683333333333, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "muse-spark-1.1-xhigh", + "complexity": "standard", + "cost_per_completed_task_usd": 0.11754979521105281, + "mean_request_cost_usd": 0.08869942363112392, + "quality_lower_bound": 0.6977158499604186, + "quality_score": 0.7415600000000001, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "muse-spark-1.1-xhigh", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.02963064556813349, + "mean_request_cost_usd": 0.027700000000000002, + "quality_lower_bound": 0.8858440022350587, + "quality_score": 0.9521560000000001, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "muse-spark-1.2-xhigh": { + "benchmark_model_id": "muse-spark-1.2-xhigh", + "identity": { + "benchmark_model_id": "muse-spark-1.2-xhigh", + "family_id": "muse-spark-1.2", + "is_routable": true, + "litellm_model_keys": [ + "meta/muse-spark-1.2" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "muse-spark-1.2-xhigh", + "complexity": "complex", + "cost_per_completed_task_usd": 2.2446638986761065, + "mean_request_cost_usd": 1.3085035714285715, + "quality_lower_bound": 0.49583582646379193, + "quality_score": 0.5833516666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "muse-spark-1.2-xhigh", + "complexity": "simple", + "cost_per_completed_task_usd": 0.08752468055455027, + "mean_request_cost_usd": 0.0718907514450867, + "quality_lower_bound": 0.7563657200849929, + "quality_score": 0.8214283333333333, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "muse-spark-1.2-xhigh", + "complexity": "standard", + "cost_per_completed_task_usd": 0.14910282543392475, + "mean_request_cost_usd": 0.12378587896253601, + "quality_lower_bound": 0.7605464624006388, + "quality_score": 0.8168583333333334, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "muse-spark-1.2-xhigh", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.03149434519589871, + "mean_request_cost_usd": 0.03006685552407932, + "quality_lower_bound": 0.9246072037939199, + "quality_score": 0.964118, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "muse-spark-1.3-xhigh": { + "benchmark_model_id": "muse-spark-1.3-xhigh", + "identity": { + "benchmark_model_id": "muse-spark-1.3-xhigh", + "family_id": "muse-spark-1.3", + "is_routable": true, + "litellm_model_keys": [ + "meta/muse-spark-1.3" + ], + "reasoning_effort": "xhigh", + "required_parameters": { + "reasoning_effort": "xhigh" + } + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "muse-spark-1.3-xhigh", + "complexity": "complex", + "cost_per_completed_task_usd": 0.7427179491890616, + "mean_request_cost_usd": 0.46920178571428567, + "quality_lower_bound": 0.5469604828322624, + "quality_score": 0.635605, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "muse-spark-1.3-xhigh", + "complexity": "simple", + "cost_per_completed_task_usd": 0.11119831208269634, + "mean_request_cost_usd": 0.09694190751445086, + "quality_lower_bound": 0.8175533481544565, + "quality_score": 0.8708016666666666, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "muse-spark-1.3-xhigh", + "complexity": "standard", + "cost_per_completed_task_usd": 0.236816088515293, + "mean_request_cost_usd": 0.20220605187319884, + "quality_lower_bound": 0.7725155351503601, + "quality_score": 0.8366733333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "muse-spark-1.3-xhigh", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.05304216893759997, + "mean_request_cost_usd": 0.05199036827195467, + "quality_lower_bound": 0.9578201313476288, + "quality_score": 0.980118, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "nemotron-3-ultra-550b-a55b": { + "benchmark_model_id": "nemotron-3-ultra-550b-a55b", + "identity": { + "benchmark_model_id": "nemotron-3-ultra-550b-a55b", + "family_id": "nemotron-3-ultra-550b-a55b", + "is_routable": true, + "litellm_model_keys": [ + "together_ai/nvidia/nemotron-3-ultra-550b-a55b" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "nemotron-3-ultra-550b-a55b", + "complexity": "complex", + "cost_per_completed_task_usd": 1.9619067092568285, + "mean_request_cost_usd": 0.9654959821428571, + "quality_lower_bound": 0.3272957964397429, + "quality_score": 0.47136666666666666, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "nemotron-3-ultra-550b-a55b", + "complexity": "simple", + "cost_per_completed_task_usd": 0.11781526781482156, + "mean_request_cost_usd": 0.08621878612716763, + "quality_lower_bound": 0.6666168118964786, + "quality_score": 0.7245983333333333, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "nemotron-3-ultra-550b-a55b", + "complexity": "standard", + "cost_per_completed_task_usd": 0.23389751572539627, + "mean_request_cost_usd": 0.15744121037463976, + "quality_lower_bound": 0.46327769300356714, + "quality_score": 0.64712, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "nemotron-3-ultra-550b-a55b", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.048535238667516814, + "mean_request_cost_usd": 0.04624362606232294, + "quality_lower_bound": 0.9080300638083402, + "quality_score": 0.949568, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "ox-alpha-max": { + "benchmark_model_id": "ox-alpha-max", + "identity": { + "benchmark_model_id": "ox-alpha-max", + "family_id": "ox-alpha-max", + "is_routable": false, + "litellm_model_keys": [], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "ox-alpha-max", + "complexity": "complex", + "cost_per_completed_task_usd": 0.0, + "mean_request_cost_usd": 0.0, + "quality_lower_bound": 0.4728252881130144, + "quality_score": 0.5206316666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "ox-alpha-max", + "complexity": "simple", + "cost_per_completed_task_usd": 0.0, + "mean_request_cost_usd": 0.0, + "quality_lower_bound": 0.7494282022616836, + "quality_score": 0.776335, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "ox-alpha-max", + "complexity": "standard", + "cost_per_completed_task_usd": 0.0, + "mean_request_cost_usd": 0.0, + "quality_lower_bound": 0.4806994442892879, + "quality_score": 0.59645, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "ox-alpha-max", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.0, + "mean_request_cost_usd": 0.0, + "quality_lower_bound": 0.8110446370636408, + "quality_score": 0.9130179999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "qwen3.6-27b": { + "benchmark_model_id": "qwen3.6-27b", + "identity": { + "benchmark_model_id": "qwen3.6-27b", + "family_id": "qwen3.6-27b", + "is_routable": true, + "litellm_model_keys": [ + "deepinfra/Qwen/Qwen3.6-27B", + "groq/qwen/qwen3.6-27b", + "libertai/qwen3.6-27b", + "novita/qwen/qwen3.6-27b", + "wandb/Qwen/Qwen3.6-27B" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "qwen3.6-27b", + "complexity": "complex", + "cost_per_completed_task_usd": 0.669680507715018, + "mean_request_cost_usd": 0.28706339285714283, + "quality_lower_bound": 0.33700489570364733, + "quality_score": 0.42367166666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "qwen3.6-27b", + "complexity": "simple", + "cost_per_completed_task_usd": 0.10038919796016793, + "mean_request_cost_usd": 0.07057976878612716, + "quality_lower_bound": 0.6166994304085625, + "quality_score": 0.6917033333333333, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "qwen3.6-27b", + "complexity": "standard", + "cost_per_completed_task_usd": 0.2490187123609404, + "mean_request_cost_usd": 0.14150086455331412, + "quality_lower_bound": 0.5143997318861502, + "quality_score": 0.57763, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "qwen3.6-27b", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.08040824423766153, + "mean_request_cost_usd": 0.07416317280453258, + "quality_lower_bound": 0.8421442118714418, + "quality_score": 0.9207259999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "qwen3.6-plus": { + "benchmark_model_id": "qwen3.6-plus", + "identity": { + "benchmark_model_id": "qwen3.6-plus", + "family_id": "qwen3.6-plus", + "is_routable": true, + "litellm_model_keys": [ + "openrouter/qwen/qwen3.6-plus", + "together_ai/Qwen/Qwen3.6-Plus" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "qwen3.6-plus", + "complexity": "complex", + "cost_per_completed_task_usd": 1.129401497128953, + "mean_request_cost_usd": 0.5205209821428571, + "quality_lower_bound": 0.34601770650322305, + "quality_score": 0.4546333333333334, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "qwen3.6-plus", + "complexity": "simple", + "cost_per_completed_task_usd": 0.08555354940409925, + "mean_request_cost_usd": 0.06570491329479769, + "quality_lower_bound": 0.7053422746386699, + "quality_score": 0.7607316666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "qwen3.6-plus", + "complexity": "standard", + "cost_per_completed_task_usd": 0.1911054834997674, + "mean_request_cost_usd": 0.1208371757925072, + "quality_lower_bound": 0.5972042015012469, + "quality_score": 0.6347383333333333, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "qwen3.6-plus", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.05245686069602515, + "mean_request_cost_usd": 0.049967705382436264, + "quality_lower_bound": 0.9238234129000658, + "quality_score": 0.953352, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "qwen3.7-max": { + "benchmark_model_id": "qwen3.7-max", + "identity": { + "benchmark_model_id": "qwen3.7-max", + "family_id": "qwen3.7-max", + "is_routable": true, + "litellm_model_keys": [ + "dashscope/qwen3.7-max", + "deepinfra/Qwen/Qwen3.7-Max", + "novita/qwen/qwen3.7-max", + "qwen_ai_platform/qwen3.7-max", + "qwencloud/qwen3.7-max", + "together_ai/Qwen/Qwen3.7-Max" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "qwen3.7-max", + "complexity": "complex", + "cost_per_completed_task_usd": 0.5703545667854351, + "mean_request_cost_usd": 0.29299419642857144, + "quality_lower_bound": 0.38837958384914156, + "quality_score": 0.49846166666666664, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "qwen3.7-max", + "complexity": "simple", + "cost_per_completed_task_usd": 0.06788574957784442, + "mean_request_cost_usd": 0.05399248554913295, + "quality_lower_bound": 0.7376394401453119, + "quality_score": 0.7878633333333332, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "qwen3.7-max", + "complexity": "standard", + "cost_per_completed_task_usd": 0.2668490255694562, + "mean_request_cost_usd": 0.19285994236311238, + "quality_lower_bound": 0.6702814450728942, + "quality_score": 0.7448049999999999, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "qwen3.7-max", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.05084514622240281, + "mean_request_cost_usd": 0.04944079320113315, + "quality_lower_bound": 0.9632810438479041, + "quality_score": 0.971196, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "qwen3.8-27b": { + "benchmark_model_id": "qwen3.8-27b", + "identity": { + "benchmark_model_id": "qwen3.8-27b", + "family_id": "qwen3.8-27b", + "is_routable": true, + "litellm_model_keys": [ + "deepinfra/Qwen/Qwen3.8-27B", + "groq/qwen/qwen3.8-27b", + "wandb/Qwen/Qwen3.8-27B" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "qwen3.8-27b", + "complexity": "complex", + "cost_per_completed_task_usd": 0.5496661807028348, + "mean_request_cost_usd": 0.31892544642857146, + "quality_lower_bound": 0.5129117470838801, + "quality_score": 0.5934633333333333, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "qwen3.8-27b", + "complexity": "simple", + "cost_per_completed_task_usd": 0.019906916011630783, + "mean_request_cost_usd": 0.015753179190751445, + "quality_lower_bound": 0.6845274259530726, + "quality_score": 0.7853766666666667, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "qwen3.8-27b", + "complexity": "standard", + "cost_per_completed_task_usd": 0.035701882386432286, + "mean_request_cost_usd": 0.025812680115273778, + "quality_lower_bound": 0.6808797772846128, + "quality_score": 0.7364366666666667, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "qwen3.8-27b", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.011706166129774697, + "mean_request_cost_usd": 0.01115070821529745, + "quality_lower_bound": 0.9245333868375888, + "quality_score": 0.9493919999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "qwen3.8-flash-next": { + "benchmark_model_id": "qwen3.8-flash-next", + "identity": { + "benchmark_model_id": "qwen3.8-flash-next", + "family_id": "qwen3.8-flash-next", + "is_routable": false, + "litellm_model_keys": [], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "qwen3.8-flash-next", + "complexity": "complex", + "cost_per_completed_task_usd": 0.10951362505075977, + "mean_request_cost_usd": 0.064796875, + "quality_lower_bound": 0.5193786665291859, + "quality_score": 0.6031333333333334, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "qwen3.8-flash-next", + "complexity": "simple", + "cost_per_completed_task_usd": 0.02584506888328626, + "mean_request_cost_usd": 0.02125664739884393, + "quality_lower_bound": 0.7364134603197254, + "quality_score": 0.8229766666666666, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "qwen3.8-flash-next", + "complexity": "standard", + "cost_per_completed_task_usd": 0.05980819821649269, + "mean_request_cost_usd": 0.04285417867435159, + "quality_lower_bound": 0.6721467035011582, + "quality_score": 0.7380116666666666, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "qwen3.8-flash-next", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.012248010549845461, + "mean_request_cost_usd": 0.01176515580736544, + "quality_lower_bound": 0.9240803336709251, + "quality_score": 0.9587260000000001, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "qwen3.8-max": { + "benchmark_model_id": "qwen3.8-max", + "identity": { + "benchmark_model_id": "qwen3.8-max", + "family_id": "qwen3.8-max", + "is_routable": true, + "litellm_model_keys": [ + "dashscope/qwen3.8-max", + "deepinfra/Qwen/Qwen3.8-Max", + "novita/qwen/qwen3.8-max", + "qwen_ai_platform/qwen3.8-max", + "qwencloud/qwen3.8-max", + "scx-ai/Qwen3.8-Max" + ], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "qwen3.8-max", + "complexity": "complex", + "cost_per_completed_task_usd": 1.0621928207243518, + "mean_request_cost_usd": 0.6400959821428571, + "quality_lower_bound": 0.5407724411526618, + "quality_score": 0.6165166666666667, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "qwen3.8-max", + "complexity": "simple", + "cost_per_completed_task_usd": 0.12897158501834355, + "mean_request_cost_usd": 0.10779421965317919, + "quality_lower_bound": 0.7495593962802758, + "quality_score": 0.836235, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "qwen3.8-max", + "complexity": "standard", + "cost_per_completed_task_usd": 0.2582985023079646, + "mean_request_cost_usd": 0.20354870317002882, + "quality_lower_bound": 0.7440344791041409, + "quality_score": 0.7852100000000001, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "qwen3.8-max", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.06643655212704988, + "mean_request_cost_usd": 0.06441331444759207, + "quality_lower_bound": 0.9512541704600264, + "quality_score": 0.971274, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + }, + "smaug-agentic": { + "benchmark_model_id": "smaug-agentic", + "identity": { + "benchmark_model_id": "smaug-agentic", + "family_id": "smaug-agentic", + "is_routable": false, + "litellm_model_keys": [], + "reasoning_effort": null, + "required_parameters": {} + }, + "quality_by_complexity": { + "complex": { + "benchmark_model_id": "smaug-agentic", + "complexity": "complex", + "cost_per_completed_task_usd": 1.1112291458427248, + "mean_request_cost_usd": 0.7018790178571429, + "quality_lower_bound": 0.5608808746166866, + "quality_score": 0.6380966666666666, + "question_count": 224, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "simple": { + "benchmark_model_id": "smaug-agentic", + "complexity": "simple", + "cost_per_completed_task_usd": 0.15925327915813411, + "mean_request_cost_usd": 0.1374485549132948, + "quality_lower_bound": 0.7993092701785569, + "quality_score": 0.85933, + "question_count": 346, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "standard": { + "benchmark_model_id": "smaug-agentic", + "complexity": "standard", + "cost_per_completed_task_usd": 0.4412840229265045, + "mean_request_cost_usd": 0.31179164265129683, + "quality_lower_bound": 0.6326602100432919, + "quality_score": 0.7396249999999999, + "question_count": 347, + "source_id": "livebench-2026-06-25", + "subtask_count": 6 + }, + "trivial": { + "benchmark_model_id": "smaug-agentic", + "complexity": "trivial", + "cost_per_completed_task_usd": 0.05537554115149625, + "mean_request_cost_usd": 0.053963739376770535, + "quality_lower_bound": 0.9592649709496708, + "quality_score": 0.9803139999999999, + "question_count": 353, + "source_id": "livebench-2026-06-25", + "subtask_count": 5 + } + }, + "task_completion_speed_by_complexity": { + "complex": [], + "simple": [], + "standard": [], + "trivial": [] + } + } + }, + "provenance": { + "inputs": [ + { + "bytes": 2133018, + "path": "litellm/model_prices_and_context_window.json", + "sha256": "a5ad23f7a2d98249588a2f21a305f8e3741bba450fd6216bde505cb9c4bee98a", + "source_id": "litellm-model-registry-2026-09-03" + }, + { + "bytes": 725, + "path": "livebench/categories_2026_06_25.json", + "sha256": "dad300ad18655b69db720e1b88fc5a5eac06c5b2f0e52c2bf50f10ff057674f3", + "source_id": "livebench-2026-06-25" + }, + { + "bytes": 23394, + "path": "livebench/cost_2026_06_25.csv", + "sha256": "6ae57254eaefe9c84008e909d118f37e53271009cf97569d43506b2ae8c9b29e", + "source_id": "livebench-2026-06-25" + }, + { + "bytes": 8939, + "path": "livebench/table_2026_06_25.csv", + "sha256": "b8e6339f7fafe6df8f6ecc8b0abe3dc529b5737a6c7edd3e24ffcb36e02fd118", + "source_id": "livebench-2026-06-25" + }, + { + "bytes": 15970, + "path": "terminal-bench/submissions/2026-08-26-anthropic-claude-fable-5-max-claude-code.json", + "sha256": "487f60477db9c36bd5cdb3d50c80634233bfa53de3368cf33337a771f17aa07e", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15964, + "path": "terminal-bench/submissions/2026-08-26-anthropic-claude-opus-4-8-max-claude-code.json", + "sha256": "25aea019c65af71083bb684cd26c30c4c6eb3f0c91d4d7473c7090c1b5e1d289", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15958, + "path": "terminal-bench/submissions/2026-08-26-anthropic-claude-opus-5-max-claude-code.json", + "sha256": "06fbebf9bf42c430c49b020f86b9365f5ae772c6063bab4fce98f54e349ed80c", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15932, + "path": "terminal-bench/submissions/2026-08-26-openai-gpt-5-6-luna-max-codex.json", + "sha256": "59ae3614eca2b0c1f087b7a50d65e3fc40dd2ef0188c2fab40d32c5a50f823a7", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15924, + "path": "terminal-bench/submissions/2026-08-26-openai-gpt-5-6-sol-max-codex.json", + "sha256": "b41e5f5f15aeb7657f9d32b33a868aecab6dd6db2d48b7480638455664102c5b", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15932, + "path": "terminal-bench/submissions/2026-08-26-openai-gpt-5-6-terra-max-codex.json", + "sha256": "1c7ca98885e40ed4dc31d23391982848b2c1dd87b0595743fa242ad848f1b670", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15887, + "path": "terminal-bench/submissions/2026-08-26-xai-grok-4-5-none-grok-build.json", + "sha256": "fa618ecb274776479019af6d62c538197cbb563a44d8b8a45752dde51d603b93", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15886, + "path": "terminal-bench/submissions/2026-08-26-xai-grok-4-6-none-grok-build.json", + "sha256": "31526e5dff0e06680085bb5029700b0fddc607962e7c6aea9a911600c6b2df1a", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15943, + "path": "terminal-bench/submissions/2026-09-01-gemini-gemini-3-8-flash-high-mini-swe-agent.json", + "sha256": "835f2090e7d40d9e5c6c5483da49c5f3147a0304e615df49c71aafa859a62c50", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15943, + "path": "terminal-bench/submissions/2026-09-02-anthropic-claude-fable-5-1-max-claude-code.json", + "sha256": "cccdb936c1266db66ce922db48144ed0716c1294eec3dcc1bdcf976711c92813", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 15953, + "path": "terminal-bench/submissions/2026-09-02-gemini-gemini-3-7-flash-high-mini-swe-agent.json", + "sha256": "013ace9d0c06b97aed4227b624aff0724257b2ca44a113553a03707545efc0cc", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 405512, + "path": "terminal-bench/trials/claude-fable-5-1-max-claude-code.json", + "sha256": "eace284d655bb870eeb113298b921d1ffde2e53f3c43ecbccf619cc6b2a50dd5", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 405541, + "path": "terminal-bench/trials/claude-fable-5-max-claude-code.json", + "sha256": "f6866893713a1feafe23a2136ae6334ff5cadf6887572e94ca8e33a3cfedc4da", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 406419, + "path": "terminal-bench/trials/claude-opus-4-8-max-claude-code.json", + "sha256": "feddf6a0222d4140696f0e25d72a6186282d20eb9170908692ada19e49a3a9ec", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 404974, + "path": "terminal-bench/trials/claude-opus-5-max-claude-code.json", + "sha256": "7a1bea779168226edfd73b547528a042a26d9fd573a6af7a174f90516ede2494", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 409532, + "path": "terminal-bench/trials/gemini-3-7-flash-high-mini-swe-agent.json", + "sha256": "b674fd778f9148d19cd3e38f441c4ecf168eb0f1056f5d9e75843ed80e66f6d2", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 406835, + "path": "terminal-bench/trials/gemini-3-8-flash-high-mini-swe-agent.json", + "sha256": "791ee2577ac9b9c9100616f53af6df0945d86b835bd39d7318bb7ee15ccf1f73", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 401789, + "path": "terminal-bench/trials/gpt-5-6-luna-max-codex.json", + "sha256": "9e223e1f7b86990d4260b96ccc76044a86b08df7c90b96c9e00f8a2a150134af", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 400386, + "path": "terminal-bench/trials/gpt-5-6-sol-max-codex.json", + "sha256": "68729d3d8bfe09362752a8ac802cbf7fc3bea6768494d9cab14439dbdac1bb54", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 401701, + "path": "terminal-bench/trials/gpt-5-6-terra-max-codex.json", + "sha256": "646b7879615f7ae5a9861bbbd7271416db4734e30a646fa84993e0692a714d6d", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 399870, + "path": "terminal-bench/trials/grok-4-5-grok-build.json", + "sha256": "0745d3a91ec75cd544f8c79ca6b162e74b2a19416d9ce428315604549ff65093", + "source_id": "terminal-bench-4.0.0" + }, + { + "bytes": 399597, + "path": "terminal-bench/trials/grok-4-6-grok-build.json", + "sha256": "6323e4142ebb1c3a054b42340e8cffa7143ff6ddf480dfdcc89cb5a710660178", + "source_id": "terminal-bench-4.0.0" + } + ], + "manifest_sha256": "128c50d897e1d3f5749c5dd9aa4ec8fa5efb9d00dea983af8194a4981c25b557", + "source_revisions": { + "litellm-model-registry-2026-09-03": "828d561fcc2704eaf90617be8cdc058fccbd8508", + "livebench-2026-06-25": "62240f848c977d4202c1029191ac663498745f2f", + "terminal-bench-4.0.0": "83c7a6172d629c6575b785ab12c8db787bb2e323" + } + }, + "quality_tiers": { + "balanced": { + "label": "Balanced", + "maximum_quality_regret": 0.07 + }, + "economy": { + "label": "Economy", + "maximum_quality_regret": 0.15 + }, + "high": { + "label": "High", + "maximum_quality_regret": 0.03 + }, + "max": { + "label": "Max", + "maximum_quality_regret": 0.01 + } + }, + "reference_routes": { + "balanced": { + "balanced": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "deepseek-v4-flash-vision-exp", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.5392130921684612, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "deepseek-v4-pro-0813", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "grok-4.6", + "gemini-3.6-flash-high", + "muse-spark-1.3-xhigh", + "claude-opus-5-max-effort", + "kimi-k3", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro-0813", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "glm-5.3", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7797278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.70251553515036, + "selected_model_id": "claude-fable-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.9068668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "cost": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-flash-vision-exp", + "muse-spark-1.3-xhigh", + "qwen3.8-max", + "kimi-k3", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.1042448619785062, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "deepseek-v4-flash-vision-exp", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.5392130921684612, + "selected_model_id": "deepseek-v4-flash-vision-exp", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "grok-4.6", + "gemini-3.6-flash-high", + "muse-spark-1.3-xhigh", + "claude-opus-5-max-effort", + "kimi-k3", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.03819422168482955, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro-0813", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "glm-5.3", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7797278750259673, + "selected_model_id": "deepseek-v4-pro-0813", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "grok-4.6", + "grok-4.5", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max", + "gemini-3.1-pro-preview-high", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.14293474305698176, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.70251553515036, + "selected_model_id": "grok-4.6", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": 0.001468206744000269, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.9068668617803175, + "selected_model_id": "deepseek-v4-flash-0731", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "task_completion_speed": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 36630585.75418182, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "deepseek-v4-flash-vision-exp", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.5392130921684612, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "deepseek-v4-pro-0813", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "grok-4.6", + "gemini-3.6-flash-high", + "muse-spark-1.3-xhigh", + "claude-opus-5-max-effort", + "kimi-k3", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro-0813", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "glm-5.3", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7797278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-max-effort" + ], + "objective_value": 9369974.166258065, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.70251553515036, + "selected_model_id": "claude-fable-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.9068668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + } + }, + "economy": { + "balanced": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.4592130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "kimi-k2.7-code", + "gemini-3.7-flash-high", + "muse-spark-1.1-xhigh", + "qwen3.7-max", + "grok-4.6", + "grok-4.5", + "qwen3.6-plus", + "muse-spark-1.2-xhigh", + "gemini-3.6-flash-high", + "glm-5.2", + "gpt-5.2-2025-12-11-high", + "muse-spark-1.3-xhigh", + "gpt-5.6-luna-max", + "claude-opus-5-max-effort", + "gemini-3.5-flash-high", + "gemini-3.1-pro-preview-high", + "qwen3.8-max", + "gpt-5.2-codex", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.6997278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-opus-5-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-8-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "glm-5.2", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "grok-build-0.1", + "inkling-xhigh", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.6225155351503601, + "selected_model_id": "claude-opus-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "minimax-m3", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "gpt-5.4-nano-xhigh", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "qwen3.6-27b", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-nano-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "minimax-m3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-27b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.8268668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "cost": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "glm-5.3-flash", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "grok-4.5", + "gemini-3.7-flash-high", + "qwen3.8-27b", + "muse-spark-1.3-xhigh", + "muse-spark-1.1-xhigh", + "gemini-3.8-flash-high", + "grok-4.6", + "qwen3.8-max", + "gpt-5.6-terra-max", + "kimi-k3", + "gpt-5.4-xhigh", + "glm-5.3", + "gpt-5.5-xhigh", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.6-sol-max", + "muse-spark-1.2-xhigh", + "claude-opus-5-max-effort", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.08444897731421191, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.4592130921684613, + "selected_model_id": "glm-5.3-flash", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "kimi-k2.7-code", + "gemini-3.7-flash-high", + "muse-spark-1.1-xhigh", + "qwen3.7-max", + "grok-4.6", + "grok-4.5", + "qwen3.6-plus", + "muse-spark-1.2-xhigh", + "gemini-3.6-flash-high", + "glm-5.2", + "gpt-5.2-2025-12-11-high", + "muse-spark-1.3-xhigh", + "gpt-5.6-luna-max", + "claude-opus-5-max-effort", + "gemini-3.5-flash-high", + "gemini-3.1-pro-preview-high", + "qwen3.8-max", + "gpt-5.2-codex", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.0060579263531826775, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.6997278750259673, + "selected_model_id": "deepseek-v4-flash-0731", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "grok-build-0.1", + "deepseek-v4-flash-0731", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "muse-spark-1.1-xhigh", + "grok-4.6", + "grok-4.5", + "muse-spark-1.2-xhigh", + "glm-5.2", + "gemini-3.7-flash-high", + "muse-spark-1.3-xhigh", + "inkling-xhigh", + "gpt-5.2-codex", + "qwen3.8-max", + "gpt-5.2-2025-12-11-high", + "qwen3.7-max", + "gemini-3.1-pro-preview-high", + "gpt-5.4-xhigh", + "gpt-5.6-terra-max", + "gpt-5.5-xhigh", + "kimi-k3", + "gpt-5.6-sol-max", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "glm-5.3", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.0034918518733508376, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "glm-5.2", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "grok-build-0.1", + "inkling-xhigh", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.6225155351503601, + "selected_model_id": "grok-build-0.1", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "minimax-m3", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "gpt-5.4-nano-xhigh", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "qwen3.6-27b", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": 0.001468206744000269, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-nano-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "minimax-m3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-27b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.8268668617803175, + "selected_model_id": "deepseek-v4-flash-0731", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "task_completion_speed": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 36630585.75418182, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.4592130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "deepseek-v4-flash-vision-exp", + "kimi-k2.7-code", + "gemini-3.7-flash-high", + "muse-spark-1.1-xhigh", + "qwen3.7-max", + "grok-4.6", + "grok-4.5", + "qwen3.6-plus", + "muse-spark-1.2-xhigh", + "gemini-3.6-flash-high", + "glm-5.2", + "gpt-5.2-2025-12-11-high", + "muse-spark-1.3-xhigh", + "gpt-5.6-luna-max", + "claude-opus-5-max-effort", + "gemini-3.5-flash-high", + "gemini-3.1-pro-preview-high", + "qwen3.8-max", + "gpt-5.2-codex", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "claude-sonnet-5-xhigh-effort", + "glm-5.3", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "kimi-k2.7-code", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.6997278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-opus-5-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-8-max-effort" + ], + "objective_value": 7639667.404512195, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "glm-5.2", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.5", + "grok-4.6", + "grok-build-0.1", + "inkling-xhigh", + "kimi-k3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.6225155351503601, + "selected_model_id": "claude-opus-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-flash-0731", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-flash-0731", + "glm-5.3-flash", + "deepseek-v4-flash", + "minimax-m3", + "qwen3.8-27b", + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "gpt-5.4-nano-xhigh", + "deepseek-v4-flash-vision-exp", + "gemini-3.5-flash-lite-high", + "kimi-k2.7-code", + "grok-4.3", + "gemini-3.7-flash-high", + "gpt-5.6-luna-max", + "grok-4.6", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "gpt-5.6-terra-max", + "glm-5.2", + "gpt-5.2-codex", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "nemotron-3-ultra-550b-a55b", + "gpt-5.4-mini-xhigh", + "qwen3.7-max", + "qwen3.6-plus", + "inkling-xhigh", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "qwen3.6-27b", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-4-6-thinking-auto-medium-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-flash", + "deepseek-v4-flash-0731", + "deepseek-v4-flash-vision-exp", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.5-flash-lite-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.2", + "glm-5.3", + "glm-5.3-flash", + "gpt-5.2-2025-12-11-high", + "gpt-5.2-codex", + "gpt-5.4-mini-xhigh", + "gpt-5.4-nano-xhigh", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-luna-max", + "gpt-5.6-sol-max", + "gpt-5.6-terra-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "inkling-xhigh", + "kimi-k2.7-code", + "kimi-k3", + "minimax-m3", + "muse-spark-1.1-xhigh", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "nemotron-3-ultra-550b-a55b", + "qwen3.6-27b", + "qwen3.6-plus", + "qwen3.7-max", + "qwen3.8-27b", + "qwen3.8-max" + ], + "quality_floor": 0.8268668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + } + }, + "high": { + "balanced": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5792130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "gpt-5.6-sol-max", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "gpt-5.6-sol-max", + "gpt-5.5-xhigh", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max" + ], + "quality_floor": 0.8197278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.7425155351503601, + "selected_model_id": "claude-fable-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-pro-0813", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "grok-4.3", + "gemini-3.7-flash-high", + "grok-4.6", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "qwen3.7-max", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.9468668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "cost": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 4.002891945931923, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5792130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "gpt-5.6-sol-max", + "gpt-5.5-xhigh", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.18762597508606915, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max" + ], + "quality_floor": 0.8197278750259673, + "selected_model_id": "gpt-5.6-sol-max", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max", + "gemini-3.1-pro-preview-high", + "claude-fable-5-max-effort" + ], + "objective_value": 0.14910282543392475, + "quality_admitted_model_ids": [ + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.7425155351503601, + "selected_model_id": "muse-spark-1.2-xhigh", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "grok-4.3", + "gemini-3.7-flash-high", + "grok-4.6", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "qwen3.7-max", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": 0.012087533779340973, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.9468668617803175, + "selected_model_id": "deepseek-v4-pro-0813", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "task_completion_speed": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 36630585.75418182, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5792130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "gpt-5.6-sol-max", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "gpt-5.6-sol-max", + "gpt-5.5-xhigh", + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max" + ], + "quality_floor": 0.8197278750259673, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-max-effort" + ], + "objective_value": 9369974.166258065, + "quality_admitted_model_ids": [ + "claude-fable-5-max-effort", + "gemini-3.1-pro-preview-high", + "muse-spark-1.2-xhigh", + "muse-spark-1.3-xhigh", + "qwen3.8-max" + ], + "quality_floor": 0.7425155351503601, + "selected_model_id": "claude-fable-5-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.231", + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "deepseek-v4-pro-0813", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "deepseek-v4-pro-0813", + "deepseek-v4-pro", + "grok-4.3", + "gemini-3.7-flash-high", + "grok-4.6", + "gpt-5.2-2025-12-11-high", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.6-flash-high", + "claude-opus-5-max-effort", + "claude-opus-4-7-xhigh-effort", + "qwen3.7-max", + "muse-spark-1.3-xhigh", + "gemini-3.5-flash-high", + "kimi-k3", + "gemini-3.8-flash-high", + "gpt-5.5-xhigh", + "gpt-5.4-xhigh", + "qwen3.8-max", + "glm-5.3", + "gemini-3.1-pro-preview-high", + "claude-sonnet-5-xhigh-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-fable-5-1-max-effort", + "claude-opus-4-8-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-opus-4-5-20251101-thinking-64k-high-effort", + "claude-opus-4-6-thinking-auto-high-effort", + "claude-opus-4-7-xhigh-effort", + "claude-opus-4-8-max-effort", + "claude-opus-5-max-effort", + "claude-sonnet-5-xhigh-effort", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "gemini-3.1-pro-preview-high", + "gemini-3.5-flash-high", + "gemini-3.6-flash-high", + "gemini-3.7-flash-high", + "gemini-3.8-flash-high", + "glm-5.3", + "gpt-5.2-2025-12-11-high", + "gpt-5.4-xhigh", + "gpt-5.5-xhigh", + "gpt-5.6-sol-max", + "grok-4.3", + "grok-4.5", + "grok-4.6", + "kimi-k3", + "muse-spark-1.3-xhigh", + "qwen3.7-max", + "qwen3.8-max" + ], + "quality_floor": 0.9468668617803175, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + } + }, + "max": { + "balanced": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 0.0, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5992130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "claude-fable-5-1-max-effort", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "quality_floor": 0.8397278750259674, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "fallback_no_speed_evidence", + "fallback_reason": "No quality-admitted model has Terminal-Bench completion-time evidence.", + "objective_eligible_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "objective_value": 0.236816088515293, + "quality_admitted_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7625155351503601, + "selected_model_id": "muse-spark-1.3-xhigh", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "gemini-3.7-flash-high", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "gemini-3.7-flash-high", + "grok-4.6", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.1-pro-preview-high" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6" + ], + "quality_floor": 0.9668668617803176, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "cost": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 4.002891945931923, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5992130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": 0.38505754057873887, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "quality_floor": 0.8397278750259674, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "objective_value": 0.236816088515293, + "quality_admitted_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7625155351503601, + "selected_model_id": "muse-spark-1.3-xhigh", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": null, + "evidence_status": "direct", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "gemini-3.7-flash-high", + "grok-4.6", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.1-pro-preview-high" + ], + "objective_value": 0.024636455346966592, + "quality_admitted_model_ids": [ + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6" + ], + "quality_floor": 0.9668668617803176, + "selected_model_id": "gemini-3.7-flash-high", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + }, + "task_completion_speed": { + "complex": { + "cold_start_model_id": null, + "evidence_status": "cohort_scoped", + "fallback_reason": null, + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "objective_value": 36630585.75418182, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort" + ], + "quality_floor": 0.5992130921684613, + "selected_model_id": "claude-fable-5-1-max-effort", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": "claude-code@2.1.257", + "speed_metric": "offline_task_completion_latency" + }, + "simple": { + "cold_start_model_id": "claude-fable-5-1-max-effort", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "claude-fable-5-1-max-effort", + "claude-fable-5-max-effort" + ], + "quality_floor": 0.8397278750259674, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + }, + "standard": { + "cold_start_model_id": null, + "evidence_status": "fallback_no_speed_evidence", + "fallback_reason": "No quality-admitted model has Terminal-Bench completion-time evidence.", + "objective_eligible_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "objective_value": 0.236816088515293, + "quality_admitted_model_ids": [ + "muse-spark-1.3-xhigh" + ], + "quality_floor": 0.7625155351503601, + "selected_model_id": "muse-spark-1.3-xhigh", + "selection_mode": "snapshot_ranked", + "speed_cohort_id": null, + "speed_metric": "offline_task_completion_latency" + }, + "trivial": { + "cold_start_model_id": "gemini-3.7-flash-high", + "evidence_status": "fallback_runtime_response_speed_required", + "fallback_reason": "The offline snapshot intentionally has no provider-neutral response-latency value. LiteLLM must rerank these quality-admitted models using live TTFT and output throughput.", + "objective_eligible_model_ids": [ + "gemini-3.7-flash-high", + "grok-4.6", + "grok-4.5", + "gpt-5.6-sol-max", + "gemini-3.1-pro-preview-high" + ], + "objective_value": null, + "quality_admitted_model_ids": [ + "gemini-3.1-pro-preview-high", + "gemini-3.7-flash-high", + "gpt-5.6-sol-max", + "grok-4.5", + "grok-4.6" + ], + "quality_floor": 0.9668668617803176, + "selected_model_id": null, + "selection_mode": "runtime_response_latency", + "speed_cohort_id": null, + "speed_metric": "online_response_latency" + } + } + } + }, + "schema_version": "2.0", + "snapshot_id": "2026-09-03-v0", + "task_complexities": { + "livebench-2026-06-25": { + "AMPS_Hard": "trivial", + "code_completion": "simple", + "code_generation": "simple", + "connections": "trivial", + "consecutive_events": "standard", + "integrals_with_game": "standard", + "javascript": "complex", + "logic_with_navigation": "standard", + "math_comp": "trivial", + "olympiad": "simple", + "paraphrase": "standard", + "plot_unscrambling": "complex", + "python": "complex", + "simplify": "complex", + "spatial": "trivial", + "story_generation": "standard", + "summarize": "standard", + "tablejoin": "complex", + "tablereformat": "trivial", + "theory_of_mind": "simple", + "typescript": "complex", + "typos": "simple", + "zebra_puzzle": "simple" + }, + "terminal-bench-4.0.0": { + "terminal-bench/atrx-vep-crispr": "trivial", + "terminal-bench/batched-eval-parity": "simple", + "terminal-bench/biped-contact-dynamics": "simple", + "terminal-bench/bun-sourcemap-leak": "complex", + "terminal-bench/cad-model": "simple", + "terminal-bench/cargo-flight-dispatch": "complex", + "terminal-bench/coq-block-bound": "trivial", + "terminal-bench/ctr-optimization": "complex", + "terminal-bench/cumulative-layout-shift": "trivial", + "terminal-bench/data-anonymization": "complex", + "terminal-bench/distributed-dedup": "simple", + "terminal-bench/embedding-drift-monitor": "simple", + "terminal-bench/fin-saccr-rwa": "trivial", + "terminal-bench/foodstuff-beta-activity": "complex", + "terminal-bench/formal-crypto": "standard", + "terminal-bench/fp8-rmsnorm-gemm": "trivial", + "terminal-bench/freecad-impeller": "complex", + "terminal-bench/freecad-platform-drawing": "standard", + "terminal-bench/freecad-spring-clip": "simple", + "terminal-bench/freight-dispatch-shift": "complex", + "terminal-bench/glycan-ms2-elucidation": "complex", + "terminal-bench/gsea-proteomics": "standard", + "terminal-bench/heat-pump-warranty": "standard", + "terminal-bench/hof-topology-interpenetration": "simple", + "terminal-bench/html-js-filter": "standard", + "terminal-bench/interleaved-vigenere": "simple", + "terminal-bench/intrastat-meldung": "trivial", + "terminal-bench/jax-speedrun-gpu": "complex", + "terminal-bench/ks-solver-cpp": "standard", + "terminal-bench/kv-live-surgery": "simple", + "terminal-bench/lake-temp-glm": "complex", + "terminal-bench/layout-config-recreation": "complex", + "terminal-bench/layout-config-recreation2": "trivial", + "terminal-bench/legacy-utility-triage": "standard", + "terminal-bench/live-database-cutover": "standard", + "terminal-bench/math-eval-grader": "standard", + "terminal-bench/medical-claims-processing": "complex", + "terminal-bench/mp-checkpoint-consolidation": "trivial", + "terminal-bench/music-harmony": "complex", + "terminal-bench/mvcc-lsm-compaction": "simple", + "terminal-bench/nextjs-performance": "standard", + "terminal-bench/ontology-kg-querying": "complex", + "terminal-bench/payments-pipeline-fix": "simple", + "terminal-bench/photonic-waveguide-routing": "simple", + "terminal-bench/pretrain-shard-corruption": "standard", + "terminal-bench/production-planning": "standard", + "terminal-bench/protein-autointerp-disulfide": "complex", + "terminal-bench/react-lead-form": "trivial", + "terminal-bench/retro-console-soc": "trivial", + "terminal-bench/risk-scorer-replay": "trivial", + "terminal-bench/roy-polymorph-cn": "complex", + "terminal-bench/rs-archive-clone": "simple", + "terminal-bench/satb-audio-transcription": "standard", + "terminal-bench/session-window-debug": "complex", + "terminal-bench/sglang-qwen-burst": "standard", + "terminal-bench/shadow-relay": "trivial", + "terminal-bench/sound-change-cascade": "simple", + "terminal-bench/takens-embedding-lean": "standard", + "terminal-bench/telecom-entity-resolution": "trivial", + "terminal-bench/uefi-bootkit": "trivial", + "terminal-bench/vba-userform-port": "simple", + "terminal-bench/vf2-speedup-networkx": "simple", + "terminal-bench/vllm-deepseek-streaming": "standard", + "terminal-bench/vpp-loss-divergence": "trivial", + "terminal-bench/wal-recovery-ordering": "simple", + "terminal-bench/wdm-design": "trivial" + } + } +} diff --git a/litellm/router_strategy/complexity_router/auto_setup.py b/litellm/router_strategy/complexity_router/auto_setup.py new file mode 100644 index 00000000000..935bf723bb5 --- /dev/null +++ b/litellm/router_strategy/complexity_router/auto_setup.py @@ -0,0 +1,378 @@ +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from functools import lru_cache +from importlib.resources import files +from typing import Final, Literal, cast + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .config import ( + AutoSetupCandidate, + AutoSetupConfig, + AutoSetupObjective, + AutoSetupQualityLevel, + AutoSetupTierPolicy, + ComplexityRouterConfig, +) + +SnapshotComplexity = Literal["trivial", "simple", "standard", "complex"] + +_SNAPSHOT_FILENAME: Final = "auto_router_snapshot_v0.json" +_TIER_COMPLEXITIES: Final[Mapping[str, SnapshotComplexity]] = { + "SIMPLE": "trivial", + "MEDIUM": "simple", + "COMPLEX": "standard", + "REASONING": "complex", +} +_EASY_COMPLEXITIES: Final = frozenset(("trivial", "simple")) +_VERSION_SEPARATOR: Final = re.compile(r"(?<=\d)\.(?=\d)") + + +class SnapshotQualityTier(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + label: str + maximum_quality_regret: float = Field(ge=0, le=1) + + +class SnapshotIdentity(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + benchmark_model_id: str + family_id: str + is_routable: bool + litellm_model_keys: tuple[str, ...] + reasoning_effort: str | None + required_parameters: Mapping[str, object] + + +class SnapshotQualityProfile(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + benchmark_model_id: str + complexity: SnapshotComplexity + cost_per_completed_task_usd: float = Field(ge=0) + mean_request_cost_usd: float = Field(ge=0) + quality_lower_bound: float = Field(ge=0, le=1) + quality_score: float = Field(ge=0, le=1) + question_count: int = Field(gt=0) + source_id: str + subtask_count: int = Field(gt=0) + + +class SnapshotSpeedProfile(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + attempt_count: int = Field(gt=0) + benchmark_model_id: str + cohort_id: str + complexity: SnapshotComplexity + cost_per_completed_task_usd: float | None = Field(default=None, ge=0) + excluded_infrastructure_trials: int = Field(ge=0) + mean_attempt_cost_usd: float | None = Field(default=None, ge=0) + observed_duration_p50_ms: float = Field(ge=0) + observed_duration_p95_ms: float = Field(ge=0) + quality_lower_bound: float = Field(ge=0, le=1) + retry_adjusted_completion_ms: float | None = Field(default=None, ge=0) + source_id: str + success_probability: float = Field(ge=0, le=1) + task_count: int = Field(gt=0) + terminal_bench_model_id: str + + +class SnapshotModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + benchmark_model_id: str + identity: SnapshotIdentity + quality_by_complexity: Mapping[SnapshotComplexity, SnapshotQualityProfile] + task_completion_speed_by_complexity: Mapping[SnapshotComplexity, tuple[SnapshotSpeedProfile, ...]] + + +class AutoRouterSnapshot(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + activation_scope: Mapping[str, object] + artifact_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + complexity_policy: Mapping[str, object] + definitions: Mapping[str, str] + generated_at: str + limitations: tuple[str, ...] + models: Mapping[str, SnapshotModel] + provenance: Mapping[str, object] + quality_tiers: Mapping[AutoSetupQualityLevel, SnapshotQualityTier] + reference_routes: Mapping[str, object] + schema_version: Literal["2.0"] + snapshot_id: str + task_complexities: Mapping[str, Mapping[str, SnapshotComplexity]] + + @model_validator(mode="after") + def _validate_policy(self) -> AutoRouterSnapshot: + if frozenset(self.quality_tiers) != frozenset(("economy", "balanced", "high", "max")): + raise ValueError("Auto Router snapshot must define economy, balanced, high, and max quality levels") + if self.activation_scope.get("setup_mode") != "auto": + raise ValueError("Auto Router snapshot is not scoped to Auto setup") + return self + + +@dataclass(frozen=True, slots=True) +class _AvailableProfile: + model_name: str + benchmark_model_id: str + quality: SnapshotQualityProfile + speed: tuple[SnapshotSpeedProfile, ...] + required_parameters: Mapping[str, object] + + +def _canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +@lru_cache(maxsize=1) +def load_auto_router_snapshot() -> AutoRouterSnapshot: + parsed = cast( + object, + json.loads( + files("litellm.router_strategy.complexity_router") + .joinpath("artifacts", _SNAPSHOT_FILENAME) + .read_text(encoding="utf-8") + ), + ) + if not isinstance(parsed, dict): + raise ValueError("Auto Router snapshot must be a JSON object") + untyped = cast(dict[object, object], parsed) + if not all(isinstance(key, str) for key in untyped): + raise ValueError("Auto Router snapshot keys must be strings") + raw = cast(dict[str, object], untyped) + expected = raw.get("artifact_sha256") + if not isinstance(expected, str): + raise ValueError("Auto Router snapshot must carry an artifact_sha256") + unsigned: Final = {key: value for key, value in raw.items() if key != "artifact_sha256"} + actual: Final = hashlib.sha256(_canonical_json(unsigned).encode("utf-8")).hexdigest() + if expected != actual: + raise ValueError(f"Auto Router snapshot hash mismatch: expected {expected}, calculated {actual}") + return AutoRouterSnapshot.model_validate(raw) + + +def _normalized_model_key(value: str) -> str: + return _VERSION_SEPARATOR.sub("-", value.strip().casefold()) + + +def _matching_snapshot_models(snapshot: AutoRouterSnapshot, model_refs: Sequence[str]) -> tuple[SnapshotModel, ...]: + normalized_refs: Final = frozenset(_normalized_model_key(ref) for ref in model_refs) + return tuple( + model + for model in snapshot.models.values() + if model.identity.is_routable + and normalized_refs.intersection(_normalized_model_key(key) for key in model.identity.litellm_model_keys) + ) + + +def _available_profiles( + snapshot: AutoRouterSnapshot, + available_model_refs: Mapping[str, Sequence[str]], + complexity: SnapshotComplexity, +) -> tuple[_AvailableProfile, ...]: + profiles: Final[list[_AvailableProfile]] = [] + for model_name, refs in available_model_refs.items(): + matches = _matching_snapshot_models(snapshot, (model_name, *refs)) + if not matches: + continue + best = min( + matches, + key=lambda model: ( + -model.quality_by_complexity[complexity].quality_lower_bound, + model.quality_by_complexity[complexity].cost_per_completed_task_usd, + model.benchmark_model_id, + ), + ) + profiles.append( + _AvailableProfile( + model_name=model_name, + benchmark_model_id=best.benchmark_model_id, + quality=best.quality_by_complexity[complexity], + speed=best.task_completion_speed_by_complexity[complexity], + required_parameters=best.identity.required_parameters, + ) + ) + return tuple(profiles) + + +def _cost_rank(profiles: Sequence[_AvailableProfile]) -> tuple[_AvailableProfile, ...]: + return tuple( + sorted( + profiles, + key=lambda profile: ( + profile.quality.cost_per_completed_task_usd, + -profile.quality.quality_lower_bound, + profile.model_name, + ), + ) + ) + + +def _normalize_log(value: float, values: Sequence[float]) -> float: + logged: Final = tuple(math.log(max(item, 1e-12)) for item in values) + low: Final = min(logged) + high: Final = max(logged) + if low == high: + return 0.0 + return (math.log(max(value, 1e-12)) - low) / (high - low) + + +def _hard_task_rank( + profiles: Sequence[_AvailableProfile], objective: AutoSetupObjective +) -> tuple[tuple[_AvailableProfile, ...], str]: + cost_ranked: Final = _cost_rank(profiles) + speed_profiles: Final = tuple( + (profile, speed) + for profile in profiles + for speed in profile.speed + if speed.retry_adjusted_completion_ms is not None + ) + if not speed_profiles: + return cost_ranked, "fallback_no_speed_evidence" + cohorts: dict[str, dict[str, tuple[_AvailableProfile, SnapshotSpeedProfile]]] = {} + for profile, speed in speed_profiles: + by_model = cohorts.setdefault(speed.cohort_id, {}) + current = by_model.get(profile.model_name) + if current is None or speed.attempt_count > current[1].attempt_count: + by_model[profile.model_name] = (profile, speed) + measured_profiles: Final = tuple({profile.model_name: profile for profile, _ in speed_profiles}.values()) + anchor: Final = max( + measured_profiles, + key=lambda profile: (profile.quality.quality_lower_bound, -profile.quality.cost_per_completed_task_usd), + ) + anchor_cohorts: Final = tuple(cohort_id for cohort_id, by_model in cohorts.items() if anchor.model_name in by_model) + selected_cohort = max( + anchor_cohorts, + key=lambda cohort_id: ( + len(cohorts[cohort_id]), + sum(speed.attempt_count for _, speed in cohorts[cohort_id].values()), + cohort_id, + ), + ) + comparable = tuple(cohorts[selected_cohort].values()) + if objective == "task_completion_speed": + return tuple( + profile + for profile, _ in sorted( + comparable, + key=lambda item: ( + item[1].retry_adjusted_completion_ms or math.inf, + -item[0].quality.quality_lower_bound, + item[0].model_name, + ), + ) + ), "cohort_scoped" + balanced_comparable = tuple(item for item in comparable if item[1].cost_per_completed_task_usd is not None) + if not balanced_comparable: + return cost_ranked, "fallback_no_comparable_cost_evidence" + costs: Final = tuple(cast(float, speed.cost_per_completed_task_usd) for _, speed in balanced_comparable) + times: Final = tuple(cast(float, speed.retry_adjusted_completion_ms) for _, speed in balanced_comparable) + return tuple( + profile + for profile, _ in sorted( + balanced_comparable, + key=lambda item: ( + math.hypot( + _normalize_log(cast(float, item[1].cost_per_completed_task_usd), costs), + _normalize_log(cast(float, item[1].retry_adjusted_completion_ms), times), + ), + -item[0].quality.quality_lower_bound, + item[0].model_name, + ), + ) + ), "cohort_scoped" + + +def _policy_candidate(profile: _AvailableProfile) -> AutoSetupCandidate: + return AutoSetupCandidate( + model_name=profile.model_name, + benchmark_model_id=profile.benchmark_model_id, + quality_lower_bound=profile.quality.quality_lower_bound, + cost_per_completed_task_usd=profile.quality.cost_per_completed_task_usd, + ) + + +def _tier_policy( + profiles: Sequence[_AvailableProfile], + complexity: SnapshotComplexity, + objective: AutoSetupObjective, +) -> AutoSetupTierPolicy: + cost_ranked: Final = _cost_rank(profiles) + if objective == "cost": + return AutoSetupTierPolicy( + selection_mode="snapshot_ranked", + candidates=tuple(_policy_candidate(profile) for profile in cost_ranked), + evidence_status="direct", + ) + if complexity in _EASY_COMPLEXITIES: + return AutoSetupTierPolicy( + selection_mode="runtime_response_latency", + candidates=tuple(_policy_candidate(profile) for profile in cost_ranked), + cold_start_model=cost_ranked[0].model_name, + evidence_status="runtime_response_speed_required", + ) + ranked, evidence_status = _hard_task_rank(profiles, objective) + return AutoSetupTierPolicy( + selection_mode="snapshot_ranked", + candidates=tuple(_policy_candidate(profile) for profile in ranked), + evidence_status=evidence_status, + ) + + +def build_auto_setup_config( + *, + snapshot: AutoRouterSnapshot, + available_model_refs: Mapping[str, Sequence[str]], + quality_level: AutoSetupQualityLevel, + optimize_for: AutoSetupObjective, +) -> ComplexityRouterConfig: + """Recompute quality gates and rankings over only the caller's available groups.""" + + maximum_regret: Final = snapshot.quality_tiers[quality_level].maximum_quality_regret + policies: Final[dict[str, AutoSetupTierPolicy]] = {} + tiers: Final[dict[str, list[str]]] = {} + tier_model_configs: Final[dict[str, list[dict[str, object]]]] = {} + for tier_name, complexity in _TIER_COMPLEXITIES.items(): + profiles = _available_profiles(snapshot, available_model_refs, complexity) + if not profiles: + raise ValueError(f"None of the available model groups has {complexity} evidence in this snapshot") + best_quality = max(profile.quality.quality_lower_bound for profile in profiles) + floor = max(0.0, best_quality - maximum_regret) + admitted = tuple(profile for profile in profiles if profile.quality.quality_lower_bound >= floor) + policy = _tier_policy(admitted, complexity, optimize_for) + policies[tier_name] = policy + tiers[tier_name] = [candidate.model_name for candidate in policy.candidates] + parameterized: list[dict[str, object]] = [ + { + "model_name": profile.model_name, + "litellm_params": dict(profile.required_parameters), + } + for profile in admitted + if profile.model_name in frozenset(tiers[tier_name]) and profile.required_parameters + ] + if parameterized: + tier_model_configs[tier_name] = parameterized + + return ComplexityRouterConfig.model_validate( + { + "tiers": tiers, + "tier_model_configs": tier_model_configs, + "classifier_type": "heuristic_v2", + "auto_setup": AutoSetupConfig( + snapshot_id=snapshot.snapshot_id, + snapshot_sha256=snapshot.artifact_sha256, + quality_level=quality_level, + optimize_for=optimize_for, + tier_policies=policies, + ).model_dump(mode="json"), + } + ) diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 17e3d1256d0..17b814e2f56 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1322,6 +1322,13 @@ class ComplexityRouter(CustomLogger): if tier is not None: tier_name: Final = _tier_name(tier) decision["tier"] = tier_name + if self.config.auto_setup is not None: + auto_policy: Final = self.config.auto_setup.tier_policies.get(tier_name) + if auto_policy is not None: + decision["auto_setup_snapshot_id"] = self.config.auto_setup.snapshot_id + decision["auto_setup_quality_level"] = self.config.auto_setup.quality_level + decision["auto_setup_optimize_for"] = self.config.auto_setup.optimize_for + decision["auto_setup_selection_mode"] = auto_policy.selection_mode if not self.config.has_custom_tiers: label = self.config.tier_label(ComplexityTier(tier_name)) if label != tier_name: @@ -1815,6 +1822,37 @@ class ComplexityRouter(CustomLogger): raise ValueError(f"Empty model pool for tier {tier_key}") return random.choice(model) + async def _pick_from_auto_setup_policy(self, tier_key: str, pool: Sequence[str]) -> str | None: + setup: Final = self.config.auto_setup + if setup is None: + return None + policy: Final = setup.tier_policies.get(tier_key) + if policy is None: + return None + available: Final = tuple(dict.fromkeys(pool)) + if not available: + raise ValueError(f"No models configured for tier {tier_key}") + ranked: Final = tuple( + candidate.model_name for candidate in policy.candidates if candidate.model_name in frozenset(available) + ) + if policy.selection_mode == "snapshot_ranked": + return ranked[0] if ranked else available[0] + + from .response_latency import select_runtime_response_model + + cold_start: Final = policy.cold_start_model + if cold_start is None: + raise ValueError(f"Auto setup runtime response-latency policy for {tier_key} has no cold-start model") + return await select_runtime_response_model( + router_cache=self.litellm_router_instance.cache, + router_model_name=self.model_name, + tier=tier_key, + candidates=policy.candidates, + available_models=available, + cold_start_model=cold_start, + objective=setup.optimize_for, + ) + def _tier_pools(self) -> dict[str, list[str]]: return {tier: (models if isinstance(models, list) else [models]) for tier, models in self.config.tiers.items()} @@ -1826,17 +1864,25 @@ class ComplexityRouter(CustomLogger): request_kwargs: dict, allowed_models: tuple[str, ...] | None = None, ) -> str: + tier_key = _tier_name(tier) if not self.config.plugins: - if allowed_models is not None: - return self._pick_from_tier_value(allowed_models, _tier_name(tier)) - return self.get_model_for_tier(tier) + configured = self.config.tiers.get(tier_key) + if configured is None: + return self.get_model_for_tier(tier) + full_pool = (configured,) if isinstance(configured, str) else tuple(configured) + pool = ( + tuple(model for model in full_pool if model in allowed_models) + if allowed_models is not None + else full_pool + ) + auto_pick: Final = await self._pick_from_auto_setup_policy(tier_key, pool) + return auto_pick if auto_pick is not None else self._pick_from_tier_value(pool, tier_key) from litellm.types.router import RoutingContext - tier_key: Final = _tier_name(tier) metadata_key: Final = get_metadata_variable_name_from_kwargs(request_kwargs) - full_pool: Final = tuple(self._tier_pools().get(tier_key, ())) - pool: Final = ( + full_pool = tuple(self._tier_pools().get(tier_key, ())) + pool = ( tuple(model for model in full_pool if model in allowed_models) if allowed_models is not None else full_pool ) if not pool: @@ -1860,7 +1906,8 @@ class ComplexityRouter(CustomLogger): # silently bypassed. Raise instead, matching the Router-level plugin # pipeline's own fail-closed behavior for the same situation. raise ValueError(f"No candidate models left for tier {tier_key} after routing-plugin filtering") - return self._pick_from_tier_value(context.candidate_models, tier_key) + auto_pick: Final = await self._pick_from_auto_setup_policy(tier_key, context.candidate_models) + return auto_pick if auto_pick is not None else self._pick_from_tier_value(context.candidate_models, tier_key) def _ensure_adaptive_router(self) -> Any | None: if not self.config.adaptive: diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 70c1b281e31..e1f78e3d969 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -27,6 +27,81 @@ class ComplexityTier(str, Enum): REASONING = "REASONING" +AutoSetupQualityLevel = Literal["economy", "balanced", "high", "max"] +AutoSetupObjective = Literal["cost", "task_completion_speed", "balanced"] +AutoSetupSelectionMode = Literal["snapshot_ranked", "runtime_response_latency"] + + +class AutoSetupCandidate(BaseModel): + """One benchmark-backed model group in an Auto-generated tier policy.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + model_name: str = Field(min_length=1) + benchmark_model_id: str = Field(min_length=1) + quality_lower_bound: float = Field(ge=0, le=1) + cost_per_completed_task_usd: float = Field(ge=0) + + +class AutoSetupTierPolicy(BaseModel): + """How an Auto-generated tier chooses from its quality-admitted model groups.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + selection_mode: AutoSetupSelectionMode + candidates: tuple[AutoSetupCandidate, ...] = Field(min_length=1) + cold_start_model: str | None = None + evidence_status: str = Field(min_length=1) + + @model_validator(mode="after") + def _validate_candidates(self) -> "AutoSetupTierPolicy": + names: Final = tuple(candidate.model_name for candidate in self.candidates) + if len(names) != len(frozenset(names)): + raise ValueError("Auto setup tier candidates must use distinct model_name values") + if self.selection_mode == "runtime_response_latency": + if self.cold_start_model not in names: + raise ValueError("A runtime response-latency policy needs a cold_start_model in candidates") + elif self.cold_start_model is not None: + raise ValueError("cold_start_model is only valid for runtime_response_latency") + return self + + +class AutoSetupConfig(BaseModel): + """Provenance and selection policy stamped only by zero-setup Auto creation.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + schema_version: Literal["1"] = "1" + snapshot_id: str = Field(min_length=1) + snapshot_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + quality_level: AutoSetupQualityLevel + optimize_for: AutoSetupObjective + tier_policies: Mapping[str, AutoSetupTierPolicy] + + @field_validator("tier_policies", mode="before") + @classmethod + def _freeze_tier_policies(cls, value: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType(dict(value)) + + @field_serializer("tier_policies") + def _serialize_tier_policies(self, value: Mapping[str, AutoSetupTierPolicy]) -> Mapping[str, AutoSetupTierPolicy]: + return dict(value) # mutable-ok: Pydantic JSON serialization requires a concrete mapping + + @model_validator(mode="after") + def _validate_tier_policies(self) -> "AutoSetupConfig": + expected: Final = frozenset(tier.value for tier in TIER_SEVERITY_ORDER) + actual: Final = frozenset(self.tier_policies) + if actual != expected: + raise ValueError( + f"Auto setup tier_policies must define exactly {', '.join(tier.value for tier in TIER_SEVERITY_ORDER)}" + ) + if self.optimize_for == "cost" and any( + policy.selection_mode != "snapshot_ranked" for policy in self.tier_policies.values() + ): + raise ValueError("Cost-optimized Auto setup cannot enable runtime response-latency selection") + return self + + class ClassificationRubric(str, Enum): """Which calibration examples, and for BUSINESS which tier criteria, the built-in classifier rubric carries.""" @@ -517,6 +592,14 @@ class ComplexityRouterConfig(BaseModel): default_factory=dict, ) + auto_setup: AutoSetupConfig | None = Field( + default=None, + description=( + "Benchmark snapshot and per-tier selection policy generated by the zero-setup Auto flow. " + "Absent on manually configured routers, whose pool selection behavior is unchanged" + ), + ) + tier_definitions: tuple[TierDefinition, ...] | None = Field( default=None, description=( @@ -1422,6 +1505,24 @@ class ComplexityRouterConfig(BaseModel): ) return self + @model_validator(mode="after") + def _validate_auto_setup(self) -> "ComplexityRouterConfig": + setup: Final = self.auto_setup + if setup is None: + return self + if self.adaptive: + raise ValueError("auto_setup and adaptive=True cannot both select from the same tier pool") + if self.tier_definitions is not None: + raise ValueError("auto_setup requires the built-in four-tier complexity ladder") + for tier_name, policy in setup.tier_policies.items(): + configured = self.tiers.get(tier_name) + configured_names = frozenset((configured,)) if isinstance(configured, str) else frozenset(configured or ()) + policy_names = frozenset(candidate.model_name for candidate in policy.candidates) + if not policy_names.issubset(configured_names): + missing = ", ".join(sorted(policy_names - configured_names)) + raise ValueError(f"auto_setup tier {tier_name} references models not present in tiers: {missing}") + return self + @model_validator(mode="after") def _validate_tier_param_placement(self) -> "ComplexityRouterConfig": """Reject a router setting written into a tier entry's request params. diff --git a/litellm/router_strategy/complexity_router/response_latency.py b/litellm/router_strategy/complexity_router/response_latency.py new file mode 100644 index 00000000000..eb33d37f73c --- /dev/null +++ b/litellm/router_strategy/complexity_router/response_latency.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import hashlib +import math +import statistics +from collections.abc import Mapping, Sequence +from datetime import datetime, timedelta +from typing import Final, Protocol, cast + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from litellm._logging import verbose_router_logger + +from .config import AutoSetupCandidate, AutoSetupObjective + +_CACHE_PREFIX: Final = "auto_setup_response_latency:v1" +_CACHE_TTL_SECONDS: Final = 7 * 24 * 60 * 60 +_MAX_SAMPLES_PER_MODEL: Final = 20 +_MAX_EXPECTED_OUTPUT_SAMPLES: Final = 100 +_MIN_SAMPLES_PER_MODEL: Final = 2 + + +class ResponseLatencyCache(Protocol): + async def async_get_cache( + self, + key: str, + parent_otel_span: object | None = None, + local_only: bool = False, + **kwargs: object, + ) -> object: ... + + async def async_set_cache( + self, + key: str, + value: object, + local_only: bool = False, + **kwargs: object, + ) -> None: ... + + +class ResponseLatencySample(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + ttft_ms: float = Field(ge=0) + output_tokens_per_ms: float = Field(gt=0) + + +class ResponseLatencyState(BaseModel): + model_config = ConfigDict(extra="forbid") + + samples_by_model: dict[str, list[ResponseLatencySample]] = Field(default_factory=dict) + visible_output_tokens: list[int] = Field(default_factory=list) + + +def _cache_key(router_model_name: str, tier: str) -> str: + namespace: Final = hashlib.sha256(router_model_name.encode("utf-8")).hexdigest()[:24] + return f"{_CACHE_PREFIX}:{namespace}:{tier}" + + +def _state(value: object) -> ResponseLatencyState: + try: + return ResponseLatencyState.model_validate(value or {}) + except ValidationError: + return ResponseLatencyState() + + +def _candidate_order( + candidates: Sequence[AutoSetupCandidate], + available_models: Sequence[str], + cold_start_model: str, +) -> tuple[str, ...]: + available: Final = frozenset(available_models) + ranked: Final = tuple(candidate.model_name for candidate in candidates if candidate.model_name in available) + extras: Final = tuple(model for model in available_models if model not in frozenset(ranked)) + ordered: Final = tuple(dict.fromkeys((*ranked, *extras))) + if cold_start_model not in ordered: + return ordered + return (cold_start_model, *(model for model in ordered if model != cold_start_model)) + + +def _median_prediction_ms(samples: Sequence[ResponseLatencySample], expected_output_tokens: float) -> float: + return statistics.median( + sample.ttft_ms + expected_output_tokens / sample.output_tokens_per_ms for sample in samples + ) + + +def _normalized_log(value: float, values: Sequence[float]) -> float: + logged: Final = tuple(math.log(max(item, 1e-12)) for item in values) + low: Final = min(logged) + high: Final = max(logged) + if low == high: + return 0.0 + return (math.log(max(value, 1e-12)) - low) / (high - low) + + +def _best_observed_model( + state: ResponseLatencyState, + candidates: Sequence[AutoSetupCandidate], + ordered_models: Sequence[str], + objective: AutoSetupObjective, +) -> str: + expected_tokens: Final = statistics.median(state.visible_output_tokens) if state.visible_output_tokens else 1.0 + latencies: Final = { + model: _median_prediction_ms(state.samples_by_model[model], expected_tokens) for model in ordered_models + } + if objective == "task_completion_speed": + return min(ordered_models, key=lambda model: (latencies[model], ordered_models.index(model))) + + configured_costs: Final = {candidate.model_name: candidate.cost_per_completed_task_usd for candidate in candidates} + known_costs: Final = tuple(configured_costs.values()) + conservative_unknown_cost: Final = max(known_costs) if known_costs else 1.0 + costs: Final = {model: configured_costs.get(model, conservative_unknown_cost) for model in ordered_models} + latency_values: Final = tuple(latencies.values()) + cost_values: Final = tuple(costs.values()) + return min( + ordered_models, + key=lambda model: ( + math.hypot( + _normalized_log(latencies[model], latency_values), + _normalized_log(costs[model], cost_values), + ), + ordered_models.index(model), + ), + ) + + +async def select_runtime_response_model( + *, + router_cache: ResponseLatencyCache, + router_model_name: str, + tier: str, + candidates: Sequence[AutoSetupCandidate], + available_models: Sequence[str], + cold_start_model: str, + objective: AutoSetupObjective, +) -> str: + """Explore quality-admitted groups, then minimize equal-output response time.""" + + ordered: Final = _candidate_order(candidates, available_models, cold_start_model) + if not ordered: + raise ValueError(f"No Auto setup candidates remain for tier {tier}") + try: + cached = await router_cache.async_get_cache(key=_cache_key(router_model_name, tier)) + state: Final = _state(cached) + except Exception as exc: # noqa: BLE001 -- optional observations must never make a request unroutable + verbose_router_logger.warning("Auto setup response-latency cache read failed: %s", exc) + return ordered[0] + + counts: Final = {model: len(state.samples_by_model.get(model, ())) for model in ordered} + least_observed: Final = min(counts.values()) + if least_observed < _MIN_SAMPLES_PER_MODEL: + return next(model for model in ordered if counts[model] == least_observed) + return _best_observed_model(state, candidates, ordered, objective) + + +def _seconds(value: object) -> float | None: + if isinstance(value, datetime): + return value.timestamp() + if isinstance(value, timedelta): + return value.total_seconds() + if isinstance(value, bool) or not isinstance(value, int | float): + return None + return float(value) + + +def _field(value: object, key: str) -> object | None: + if isinstance(value, Mapping): + return cast(Mapping[object, object], value).get(key) + try: + return cast(object, object.__getattribute__(value, key)) + except AttributeError: + return None + + +def _visible_output_tokens(response_obj: object) -> int | None: + usage = _field(response_obj, "usage") + if usage is None: + return None + total = _field(usage, "completion_tokens") + if total is None: + total = _field(usage, "output_tokens") + if isinstance(total, bool) or not isinstance(total, int | float) or total <= 0: + return None + details = _field(usage, "completion_tokens_details") + if details is None: + details = _field(usage, "output_tokens_details") + reasoning = _field(details, "reasoning_tokens") if details is not None else None + reasoning_tokens = float(reasoning) if isinstance(reasoning, int | float) and not isinstance(reasoning, bool) else 0 + return max(1, round(float(total) - reasoning_tokens)) + + +def response_latency_sample( + kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object +) -> tuple[ResponseLatencySample, int] | None: + start: Final = _seconds(start_time) + end: Final = _seconds(end_time) + visible_tokens: Final = _visible_output_tokens(response_obj) + if start is None or end is None or end <= start or visible_tokens is None: + return None + total_ms: Final = (end - start) * 1000 + completion_start: Final = _seconds(kwargs.get("completion_start_time")) + ttft_ms = ( + (completion_start - start) * 1000 if completion_start is not None and start < completion_start < end else 0.0 + ) + generation_ms: Final = max(total_ms - ttft_ms, 1e-6) + return ResponseLatencySample( + ttft_ms=ttft_ms, + output_tokens_per_ms=visible_tokens / generation_ms, + ), visible_tokens + + +async def record_runtime_response_latency( + *, + router_cache: ResponseLatencyCache, + router_model_name: str, + tier: str, + routed_model: str, + kwargs: Mapping[str, object], + response_obj: object, + start_time: object, + end_time: object, +) -> None: + measured: Final = response_latency_sample(kwargs, response_obj, start_time, end_time) + if measured is None: + return + sample, visible_tokens = measured + key: Final = _cache_key(router_model_name, tier) + state: Final = _state(await router_cache.async_get_cache(key=key)) + samples: Final = [*state.samples_by_model.get(routed_model, ()), sample][-_MAX_SAMPLES_PER_MODEL:] + state.samples_by_model[routed_model] = samples + state.visible_output_tokens = [*state.visible_output_tokens, visible_tokens][-_MAX_EXPECTED_OUTPUT_SAMPLES:] + await router_cache.async_set_cache(key=key, value=state.model_dump(mode="json"), ttl=_CACHE_TTL_SECONDS) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 88869a1edfb..e13c297870c 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -9,7 +9,11 @@ from typing import Final, Literal, TypeAlias from pydantic import BaseModel, Field, computed_field, field_validator, model_validator -from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig +from litellm.router_strategy.complexity_router.config import ( + AutoSetupObjective, + AutoSetupQualityLevel, + ComplexityRouterConfig, +) from litellm.types.utils import StandardLoggingRoutingDecision DEFAULT_ROUTING_TEST_ROUTER_NAME: Final[str] = "auto_router_routing_test" @@ -44,6 +48,17 @@ class ComplexityRouterConfigValidationResponse(BaseModel): error: str | None = None +class AutoRouterRecommendationResponse(BaseModel): + """An editable complexity-router config generated from the caller's usable model groups.""" + + quality_level: AutoSetupQualityLevel + optimize_for: AutoSetupObjective + snapshot_id: str + snapshot_generated_at: str + matched_model_groups: tuple[str, ...] + complexity_router_config: RequestComplexityRouterConfig + + class AutoRouterRoutingTestRequest(BaseModel): """A single request to classify against a complexity-router config that need not be saved yet. diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 34c47d3f201..0d327540989 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2938,6 +2938,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): savings_baseline_model: str savings_baseline_deployment_id: str tier_litellm_params: Mapping[str, object] # writable-ok: Pydantic warns on ReadOnly TypedDict fields + auto_setup_snapshot_id: str + auto_setup_quality_level: Literal["economy", "balanced", "high", "max"] + auto_setup_optimize_for: Literal["cost", "task_completion_speed", "balanced"] + auto_setup_selection_mode: Literal["snapshot_ranked", "runtime_response_latency"] # Fields whose values quote the caller's prompt. Dropped when an operator turns message diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index c525af84511..7dceb270f85 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -18,6 +18,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.management_endpoints.auto_router_endpoints import ( + get_auto_router_recommendation, preview_auto_router_routing, ) from litellm.router import Router @@ -104,6 +105,51 @@ async def _route(prompt: str, monkeypatch: pytest.MonkeyPatch, **config_override return await _route_body({"prompt": prompt}, monkeypatch, **config_overrides) +@pytest.mark.asyncio +async def test_recommendation_uses_caller_groups_and_returns_an_editable_complexity_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm.proxy.management_endpoints.auto_router_endpoints as endpoint_module + import litellm.proxy.utils as proxy_utils + from litellm.proxy import proxy_server + + model_list = [ + { + "model_name": "small-group", + "litellm_params": {"model": "gpt-5.4-nano", "api_key": "fake"}, + }, + { + "model_name": "smart-group", + "litellm_params": {"model": "gpt-5.6-sol", "api_key": "fake"}, + }, + { + "model_name": "unavailable-group", + "litellm_params": {"model": "gpt-5.5", "api_key": "fake"}, + }, + ] + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=model_list)) + available = AsyncMock(return_value=["small-group", "smart-group"]) + monkeypatch.setattr(proxy_utils, "get_available_models_for_user", available) + authorize = AsyncMock() + monkeypatch.setattr(endpoint_module, "_authorize_router_dry_run", authorize) + + response = await get_auto_router_recommendation( + user_api_key_dict=ADMIN, + quality_level="economy", + optimize_for="task_completion_speed", + ) + + authorize.assert_awaited_once_with(user_api_key_dict=ADMIN, team_id=None) + assert set(response.matched_model_groups) <= {"small-group", "smart-group"} + assert "unavailable-group" not in response.matched_model_groups + assert response.complexity_router_config.classifier_type == "heuristic_v2" + assert response.complexity_router_config.auto_setup is not None + assert response.complexity_router_config.auto_setup.optimize_for == "task_completion_speed" + assert response.complexity_router_config.auto_setup.tier_policies["SIMPLE"].selection_mode == ( + "runtime_response_latency" + ) + + AGENTIC_MESSAGES = [ {"role": "system", "content": "You are a database migration assistant for a payments ledger"}, {"role": "user", "content": "duplicate ledger postings since the celery upgrade, same event_id twice"}, diff --git a/tests/test_litellm/router_strategy/test_complexity_router_auto_setup.py b/tests/test_litellm/router_strategy/test_complexity_router_auto_setup.py new file mode 100644 index 00000000000..4028181efe0 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_complexity_router_auto_setup.py @@ -0,0 +1,276 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from pydantic import ValidationError + +from litellm import Router +from litellm.router_strategy.complexity_router.auto_setup import ( + build_auto_setup_config, + load_auto_router_snapshot, +) +from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter +from litellm.router_strategy.complexity_router.config import ( + AutoSetupCandidate, + ComplexityRouterConfig, + ComplexityTier, +) +from litellm.router_strategy.complexity_router.response_latency import ( + record_runtime_response_latency, + response_latency_sample, + select_runtime_response_model, +) + + +def _available_refs(*model_names: str) -> dict[str, tuple[str, ...]]: + return {model_name: (model_name,) for model_name in model_names} + + +def test_snapshot_is_verified_and_quality_gate_is_recomputed_for_available_models() -> None: + snapshot = load_auto_router_snapshot() + available = _available_refs("gpt-5.6-sol", "gpt-5.4-nano") + + economy = build_auto_setup_config( + snapshot=snapshot, + available_model_refs=available, + quality_level="economy", + optimize_for="cost", + ) + maximum = build_auto_setup_config( + snapshot=snapshot, + available_model_refs=available, + quality_level="max", + optimize_for="cost", + ) + nano_only = build_auto_setup_config( + snapshot=snapshot, + available_model_refs=_available_refs("gpt-5.4-nano"), + quality_level="max", + optimize_for="cost", + ) + + assert snapshot.artifact_sha256 == "d0ea7e584dbb9ea37eb15ce20ae342612e0dfaa5801b553e71fc76818d7cef64" + assert economy.tiers["SIMPLE"] == ["gpt-5.4-nano", "gpt-5.6-sol"] + assert maximum.tiers["SIMPLE"] == ["gpt-5.6-sol"] + assert nano_only.tiers["SIMPLE"] == ["gpt-5.4-nano"] + + +def test_speed_only_changes_auto_generated_selection_policy() -> None: + config = build_auto_setup_config( + snapshot=load_auto_router_snapshot(), + available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"), + quality_level="economy", + optimize_for="task_completion_speed", + ) + + assert config.auto_setup is not None + assert config.auto_setup.tier_policies["SIMPLE"].selection_mode == "runtime_response_latency" + assert config.auto_setup.tier_policies["MEDIUM"].selection_mode == "runtime_response_latency" + assert config.auto_setup.tier_policies["COMPLEX"].selection_mode == "snapshot_ranked" + assert config.auto_setup.tier_policies["REASONING"].selection_mode == "snapshot_ranked" + assert ComplexityRouterConfig(tiers=config.tiers).auto_setup is None + + +def test_auto_setup_cannot_change_adaptive_or_custom_tier_routers() -> None: + generated = build_auto_setup_config( + snapshot=load_auto_router_snapshot(), + available_model_refs=_available_refs("gpt-5.6-sol"), + quality_level="max", + optimize_for="cost", + ) + payload = generated.model_dump(mode="json") + + with pytest.raises(ValidationError, match="auto_setup and adaptive"): + ComplexityRouterConfig.model_validate({**payload, "adaptive": True}) + with pytest.raises(ValidationError, match="built-in four-tier"): + ComplexityRouterConfig.model_validate( + { + **payload, + "tier_definitions": [ + {"name": "fast", "description": "Fast"}, + {"name": "strong", "description": "Strong"}, + ], + "tiers": {"fast": ["gpt-5.6-sol"], "strong": ["gpt-5.6-sol"]}, + "fallback_tier": "fast", + "classifier_type": "llm", + "classifier_llm_config": {"model": "gpt-5.6-sol"}, + } + ) + + +class _Cache: + def __init__(self) -> None: + self.values: dict[str, object] = {} + + async def async_get_cache( + self, + key: str, + parent_otel_span: object | None = None, + local_only: bool = False, + **kwargs: object, + ) -> object: + return self.values.get(key) + + async def async_set_cache( + self, + key: str, + value: object, + local_only: bool = False, + **kwargs: object, + ) -> None: + self.values[key] = value + + +def _candidate(model_name: str, cost: float) -> AutoSetupCandidate: + return AutoSetupCandidate( + model_name=model_name, + benchmark_model_id=f"benchmark-{model_name}", + quality_lower_bound=0.95, + cost_per_completed_task_usd=cost, + ) + + +@pytest.mark.asyncio +async def test_easy_task_speed_learns_equal_output_response_time() -> None: + cache = _Cache() + candidates = (_candidate("cheap-slow", 0.01), _candidate("fast", 0.04)) + + assert ( + await select_runtime_response_model( + router_cache=cache, + router_model_name="auto", + tier="SIMPLE", + candidates=candidates, + available_models=("cheap-slow", "fast"), + cold_start_model="cheap-slow", + objective="task_completion_speed", + ) + == "cheap-slow" + ) + + for model, end in (("cheap-slow", 0.30), ("cheap-slow", 0.30), ("fast", 0.15), ("fast", 0.15)): + await record_runtime_response_latency( + router_cache=cache, + router_model_name="auto", + tier="SIMPLE", + routed_model=model, + kwargs={"completion_start_time": 0.10}, + response_obj={ + "usage": { + "completion_tokens": 20, + "completion_tokens_details": {"reasoning_tokens": 10}, + } + }, + start_time=0.0, + end_time=end, + ) + + measured = response_latency_sample( + {"completion_start_time": 0.10}, + {"usage": {"completion_tokens": 20, "completion_tokens_details": {"reasoning_tokens": 10}}}, + 0.0, + 0.30, + ) + assert measured is not None and measured[1] == 10 + assert ( + await select_runtime_response_model( + router_cache=cache, + router_model_name="auto", + tier="SIMPLE", + candidates=candidates, + available_models=("cheap-slow", "fast"), + cold_start_model="cheap-slow", + objective="task_completion_speed", + ) + == "fast" + ) + + +@pytest.mark.asyncio +async def test_snapshot_policy_is_deterministic_but_manual_pool_keeps_random_selection() -> None: + generated = build_auto_setup_config( + snapshot=load_auto_router_snapshot(), + available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"), + quality_level="economy", + optimize_for="cost", + ) + router_instance = MagicMock() + router_instance.cache = AsyncMock() + automatic = ComplexityRouter( + model_name="auto", + litellm_router_instance=router_instance, + complexity_router_config=generated.model_dump(mode="json"), + ) + manual = ComplexityRouter( + model_name="manual", + litellm_router_instance=router_instance, + complexity_router_config={"tiers": {"SIMPLE": ["first", "second"]}}, + ) + + assert await automatic._pick_model_for_tier(ComplexityTier.SIMPLE, None, None, {}) == "gpt-5.4-nano" + assert ( + await automatic._pick_model_for_tier( + ComplexityTier.SIMPLE, + None, + None, + {}, + allowed_models=("gpt-5.6-sol",), + ) + == "gpt-5.6-sol" + ) + with patch( + "litellm.router_strategy.complexity_router.complexity_router.random.choice", return_value="second" + ) as pick: + assert await manual._pick_model_for_tier(ComplexityTier.SIMPLE, None, None, {}) == "second" + pick.assert_called_once_with(("first", "second")) + + +@pytest.mark.asyncio +async def test_live_latency_telemetry_is_recorded_only_for_an_auto_runtime_policy() -> None: + generated = build_auto_setup_config( + snapshot=load_auto_router_snapshot(), + available_model_refs=_available_refs("gpt-5.6-sol", "gpt-5.4-nano"), + quality_level="economy", + optimize_for="task_completion_speed", + ) + router = Router( + model_list=[ + { + "model_name": "smart", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": generated.model_dump(mode="json"), + "complexity_router_default_model": "gpt-5.6-sol", + }, + }, + {"model_name": "gpt-5.6-sol", "litellm_params": {"model": "gpt-5.6-sol"}}, + {"model_name": "gpt-5.4-nano", "litellm_params": {"model": "gpt-5.4-nano"}}, + ] + ) + metadata = { + "routing_decision": { + "router_model_name": "smart", + "tier": "SIMPLE", + "routed_model": "gpt-5.4-nano", + "auto_setup_selection_mode": "runtime_response_latency", + } + } + + with patch( + "litellm.router_strategy.complexity_router.response_latency.record_runtime_response_latency", + new_callable=AsyncMock, + ) as recorder: + await router._record_auto_setup_response_latency( + kwargs={"litellm_params": {"metadata": metadata}}, + completion_response={"usage": {"completion_tokens": 1}}, + start_time=0.0, + end_time=0.1, + ) + metadata["routing_decision"]["auto_setup_selection_mode"] = "snapshot_ranked" + await router._record_auto_setup_response_latency( + kwargs={"litellm_params": {"metadata": metadata}}, + completion_response={"usage": {"completion_tokens": 1}}, + start_time=0.0, + end_time=0.1, + ) + + recorder.assert_awaited_once() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation.ts new file mode 100644 index 00000000000..d13452ffd9a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation.ts @@ -0,0 +1,34 @@ +import { useQuery } from "@tanstack/react-query"; + +import { getAutoRouterRecommendation, type AutoRouterRecommendationResponse } from "@/components/networking"; +import type { AutoSetupObjective, AutoSetupQualityLevel } from "@/components/add_model/build_complexity_router_config"; +import { createQueryKeys } from "../common/queryKeysFactory"; + +const recommendationKeys = createQueryKeys("autoRouterRecommendation"); + +interface UseAutoRouterRecommendationParams { + accessToken: string; + qualityLevel: AutoSetupQualityLevel; + optimizeFor: AutoSetupObjective; + teamId?: string; + enabled: boolean; +} + +export const useAutoRouterRecommendation = ({ + accessToken, + qualityLevel, + optimizeFor, + teamId, + enabled, +}: UseAutoRouterRecommendationParams) => { + const filters = { qualityLevel, optimizeFor, ...(teamId ? { teamId } : {}) }; + const queryOptions = { + queryKey: recommendationKeys.list({ + filters, + }), + queryFn: () => getAutoRouterRecommendation(accessToken, qualityLevel, optimizeFor, teamId), + enabled: enabled && Boolean(accessToken), + staleTime: 30 * 1000, + }; + return useQuery(queryOptions); +}; diff --git a/ui/litellm-dashboard/src/components/add_model/AutoSetupControls.tsx b/ui/litellm-dashboard/src/components/add_model/AutoSetupControls.tsx new file mode 100644 index 00000000000..ab0f45bd907 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/AutoSetupControls.tsx @@ -0,0 +1,134 @@ +import type { AutoRouterRecommendationResponse } from "@/components/networking"; +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import type { AutoSetupObjective, AutoSetupQualityLevel } from "./build_complexity_router_config"; + +const OBJECTIVE_DESCRIPTION: Record = { + cost: "Uses expected cost per completed benchmark task, including failures", + task_completion_speed: "Uses live response speed for easy requests and benchmark completion time for hard ones", + balanced: "Balances cost per completed task with those same completion-time signals", +}; + +interface AutoSetupControlsProps { + qualityLevel: AutoSetupQualityLevel; + optimizeFor: AutoSetupObjective; + onQualityLevelChange: (value: AutoSetupQualityLevel) => void; + onOptimizeForChange: (value: AutoSetupObjective) => void; + recommendation?: AutoRouterRecommendationResponse; + recommendationEnabled: boolean; + recommendationPending: boolean; + recommendationFailed: boolean; + recommendationError: unknown; + requiresTeamScope: boolean; + hasAutoPolicy: boolean; + onRetry: () => void; + onRegenerate: () => void; + onUseManual: () => void; +} + +const recommendationErrorMessage = (error: unknown): string => + error instanceof Error ? error.message : "Could not build Auto setup"; + +const readyMessage = (recommendation: AutoRouterRecommendationResponse): string => { + const count = recommendation.matched_model_groups.length; + return `Ready with ${count} available model${count === 1 ? "" : "s"}. Review or edit below`; +}; + +export default function AutoSetupControls({ + qualityLevel, + optimizeFor, + onQualityLevelChange, + onOptimizeForChange, + recommendation, + recommendationEnabled, + recommendationPending, + recommendationFailed, + recommendationError, + requiresTeamScope, + hasAutoPolicy, + onRetry, + onRegenerate, + onUseManual, +}: AutoSetupControlsProps) { + return ( +
+
+
Auto setup
+
+ Builds an editable complexity router from the models available to you +
+
+
+
+ + +
+ Higher levels keep each tier closer to the smartest available model +
+
+
+ + +
{OBJECTIVE_DESCRIPTION[optimizeFor]}
+
+
+ {recommendationPending && recommendation === undefined && ( +
+ Building your router... +
+ )} + {!recommendationEnabled && requiresTeamScope && ( +
Select a team to build its Auto setup
+ )} + {recommendationFailed && recommendation === undefined && ( +
+ {recommendationErrorMessage(recommendationError)}.{" "} + +
+ )} + {recommendation && ( +
+
+ {hasAutoPolicy ? readyMessage(recommendation) : "Customized model pools now use standard manual selection"} +
+ {!hasAutoPolicy && ( + + )} +
+ )} + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 02cb0543e17..62e950c890c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -2,6 +2,7 @@ import { fireEvent, renderWithProviders, screen, within } from "../../../tests/t import userEvent from "@testing-library/user-event"; import { vi } from "vitest"; import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { detachAutoSetupAfterPoolEdit } from "./auto_setup_policy"; vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../../tests/mocks/complexityScorerDefaults"), @@ -44,6 +45,30 @@ const baseProps = { }; describe("ComplexityRouterConfig", () => { + it("keeps Auto policy for classifier edits and detaches it for model-pool edits", () => { + const automatic: ComplexityRouterConfigValue = { + ...defaultValue, + auto_setup: { + schema_version: "1", + snapshot_id: "snapshot-test", + snapshot_sha256: "a".repeat(64), + quality_level: "max", + optimize_for: "cost", + tier_policies: {}, + }, + }; + + expect(detachAutoSetupAfterPoolEdit(automatic, { ...automatic, classifier_type: "heuristic_v2" }).auto_setup).toBe( + automatic.auto_setup, + ); + expect( + detachAutoSetupAfterPoolEdit(automatic, { + ...automatic, + tiers: { ...automatic.tiers, SIMPLE: ["gpt-4"] }, + }).auto_setup, + ).toBeUndefined(); + }); + it("should render", () => { renderWithProviders(); expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 791fd1fcfbb..8e85e215619 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -48,6 +48,7 @@ import EscalationKeywords from "./EscalationKeywords"; import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules"; import SemanticKeywordMatching from "./SemanticKeywordMatching"; import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs"; +import type { AutoSetupConfigPayload } from "./build_complexity_router_config"; export type { DimensionWeights, TierBoundaries, TokenThresholds }; export type { CustomTierSet, TierRow } from "./tier_rows"; @@ -446,6 +447,8 @@ export interface ComplexityRouterConfigValue { * edit round-trip. */ tier_model_params?: TierModelParamsByTier; + /** Present only when the zero-setup Auto flow generated this editable config. */ + auto_setup?: AutoSetupConfigPayload; } /** Session affinity wins where a hand-authored config sets both, matching the backend's own `or`. */ diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 836b4c6ff3d..fc681bbf940 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -4,13 +4,14 @@ import { vi } from "vitest"; import AddAutoRouterTab from "./add_auto_router_tab"; import { toast } from "@/lib/toast"; import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; -import { getMissingTiersError } from "./build_complexity_router_config"; +import { getMissingTiersError, type AutoSetupConfigPayload } from "./build_complexity_router_config"; import { getSubmitBlockedReason } from "./add_auto_router_tab"; import { buildModelAvailability } from "@/lib/autorouter_presets"; import { testAutoRouterRouting } from "../networking"; import { ModelGroup } from "@/components/llm_calls/fetch_models"; import { AutoRouterPreset, getRequiredModelsInPreset } from "@/lib/autorouter_presets"; import { BUNDLED_PRESETS, LOADED_PRESETS_QUERY, useAutoRouterPresets } from "../../../tests/mocks/autoRouterPresets"; +import { EMPTY_RECOMMENDATION_QUERY, useAutoRouterRecommendation } from "../../../tests/mocks/autoRouterRecommendation"; vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../../tests/mocks/complexityScorerDefaults"), @@ -19,6 +20,10 @@ vi.mock( "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets", async () => await import("../../../tests/mocks/autoRouterPresets"), ); +vi.mock( + "@/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation", + async () => await import("../../../tests/mocks/autoRouterRecommendation"), +); const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS; const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key); @@ -37,6 +42,9 @@ const ALL_FAMILY_MODELS: ModelGroup[] = [ const ANTHROPIC_ONLY_MODEL = ANTHROPIC_TIERS.COMPLEX[0]; const openTemplateDropdown = (): void => { + if (!screen.queryByTestId("template-selector")) { + fireEvent.click(screen.getByRole("button", { name: /choose a template or set up manually/i })); + } fireEvent.click(screen.getByTestId("template-selector")); }; @@ -141,6 +149,86 @@ describe("AddAutoRouterTab", () => { testQueryClient.clear(); mockFetchAvailableModels.mockResolvedValue([]); mockFetchAllModelDeployments.mockResolvedValue([]); + vi.mocked(useAutoRouterRecommendation).mockReturnValue(EMPTY_RECOMMENDATION_QUERY); + }); + + it("starts with only quality and optimization choices in Auto setup", () => { + renderWithProviders(); + + expect(screen.getByTestId("auto-quality-selector")).toBeInTheDocument(); + expect(screen.getByTestId("auto-objective-selector")).toBeInTheDocument(); + expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument(); + expect(useAutoRouterRecommendation).toHaveBeenCalledWith( + expect.objectContaining({ qualityLevel: "balanced", optimizeFor: "balanced", enabled: true }), + ); + }); + + it("submits the editable policy generated for the caller's available model groups", async () => { + const user = userEvent.setup(); + const autoSetup: AutoSetupConfigPayload = { + schema_version: "1" as const, + snapshot_id: "snapshot-test", + snapshot_sha256: "a".repeat(64), + quality_level: "max" as const, + optimize_for: "task_completion_speed" as const, + tier_policies: Object.fromEntries( + ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"].map((tier) => [ + tier, + { + selection_mode: tier === "SIMPLE" || tier === "MEDIUM" ? "runtime_response_latency" : "snapshot_ranked", + candidates: [ + { + model_name: "smart-group", + benchmark_model_id: "benchmark-smart", + quality_lower_bound: 0.98, + cost_per_completed_task_usd: 0.04, + }, + ], + ...(tier === "SIMPLE" || tier === "MEDIUM" ? { cold_start_model: "smart-group" } : {}), + evidence_status: "test", + }, + ]), + ), + }; + mockFetchAvailableModels.mockResolvedValue([{ model_group: "smart-group", mode: "chat" }]); + vi.mocked(useAutoRouterRecommendation).mockReturnValue({ + data: { + quality_level: "max", + optimize_for: "task_completion_speed", + snapshot_id: "snapshot-test", + snapshot_generated_at: "2026-09-03T00:00:00Z", + matched_model_groups: ["smart-group"], + complexity_router_config: { + tiers: { + SIMPLE: ["smart-group"], + MEDIUM: ["smart-group"], + COMPLEX: ["smart-group"], + REASONING: ["smart-group"], + }, + classifier_type: "heuristic_v2", + classification_mode: "per_request", + session_affinity: false, + deployment_affinity: false, + modality_routing: false, + modality_pin_override: false, + auto_setup: autoSetup, + }, + }, + isPending: false, + isError: false, + error: null, + refetch: vi.fn(), + }); + + renderWithProviders(); + await screen.findByText(/Ready with 1 available model/); + await user.type(screen.getByPlaceholderText(/smart_router/i), "max-speed-router"); + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0].complexity_router_config).toMatchObject({ + auto_setup: autoSetup, + }); }); // Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller @@ -163,6 +251,23 @@ describe("AddAutoRouterTab", () => { expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); }); + it("blocks an implicit form submit while Auto setup is still building", async () => { + vi.mocked(getMissingTiersError).mockReturnValue(null); + vi.mocked(useAutoRouterRecommendation).mockReturnValue({ + ...EMPTY_RECOMMENDATION_QUERY, + isPending: true, + }); + const { container } = renderWithProviders(); + + fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "early-router" } }); + fireEvent.submit(container.querySelector("form")!); + + await waitFor(() => + expect(toast.fromError).toHaveBeenCalledWith("Building the best setup for your available models"), + ); + expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); + }); + it("still flags the router name once the config no longer blocks the submit", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); @@ -354,22 +459,19 @@ describe("AddAutoRouterTab", () => { }); it("blocks the submit when a team admin has not picked a team", async () => { - const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); renderWithProviders( , ); - await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router"); - await user.click(screen.getByRole("button", { name: /add auto router/i })); - - expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument(); + expect(screen.getByText("Select a team to build its Auto setup")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); }); // The shared dropdown emits null on clear while this form's schema wants a string, so the - // form maps null back to "": the user sees the pick-a-team message, not a zod type error. + // form maps null back to "": Auto setup returns to its explicit pick-a-team state. it("treats a team picked and then cleared like no team at all", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); @@ -381,9 +483,8 @@ describe("AddAutoRouterTab", () => { await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router"); await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1"); await user.click(screen.getByTestId("team-dropdown-clear")); - await user.click(screen.getByRole("button", { name: /add auto router/i })); - - expect(await screen.findByText("Please select a team to continue")).toBeInTheDocument(); + expect(screen.getByText("Select a team to build its Auto setup")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); }); @@ -750,8 +851,8 @@ describe("AddAutoRouterTab", () => { renderWithProviders(); - expect(await screen.findByText("Could not load available models.")).toBeInTheDocument(); openTemplateDropdown(); + expect(await screen.findByText("Could not load available models.")).toBeInTheDocument(); const anthropicOption = optionByLabel("Anthropic Family")!; expect(isOptionDisabled(anthropicOption)).toBe(true); expect(anthropicOption).toHaveTextContent(/Cannot verify these models are available/); @@ -1227,6 +1328,7 @@ describe("preset catalog fetch states", () => { } as never); renderWithProviders(); + openTemplateDropdown(); expect(screen.getByText("Loading templates...")).toBeInTheDocument(); }); @@ -1240,9 +1342,8 @@ describe("preset catalog fetch states", () => { } as never); renderWithProviders(); - expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument(); - openTemplateDropdown(); + expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument(); const options = screen.queryAllByRole("option"); expect(options).toHaveLength(1); expect(options[0]).toHaveTextContent("Custom Configuration"); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index c335beade91..f2f16605dc9 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import { useQuery } from "@tanstack/react-query"; import { useWatch } from "react-hook-form"; import { ChevronDown, ChevronRight } from "lucide-react"; @@ -30,10 +30,13 @@ import ComplexityRouterConfig, { DEFAULT_DEPLOYMENT_AFFINITY, DEFAULT_TIER_DISTANCE_PENALTY, } from "./ComplexityRouterConfig"; +import { detachAutoSetupAfterPoolEdit, getRecommendationBlockedReason, type SetupMode } from "./auto_setup_policy"; import { KeywordTierRule } from "./KeywordTierRules"; import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching"; import { + AutoSetupObjective, + AutoSetupQualityLevel, BuildComplexityRouterConfigParams, buildComplexityRouterConfig, getKeywordTierRulesError, @@ -63,7 +66,9 @@ import { AutoRouterPreset, } from "@/lib/autorouter_presets"; import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets"; +import { useAutoRouterRecommendation } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterRecommendation"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import AutoSetupControls from "./AutoSetupControls"; interface AddAutoRouterTabProps { handleOk: () => void; @@ -181,6 +186,10 @@ const AddAutoRouterTab: React.FC = ({ const watchedName = useWatch({ control: form.control, name: "auto_router_name" }); const watchedTeamId = useWatch({ control: form.control, name: "team_id" }); const [modelAccessGroups, setModelAccessGroups] = useState([]); + const [setupMode, setSetupMode] = useState("auto"); + const [qualityLevel, setQualityLevel] = useState("balanced"); + const [optimizeFor, setOptimizeFor] = useState("balanced"); + const appliedRecommendationKey = useRef(null); const [complexityRouterConfig, setComplexityRouterConfig] = useState({ tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, @@ -267,6 +276,22 @@ const AddAutoRouterTab: React.FC = ({ [modelInfo], ); + const recommendationEnabled = setupMode === "auto" && (!requiresTeamScope || Boolean(watchedTeamId)); + const recommendationParams = { + accessToken, + qualityLevel, + optimizeFor, + teamId: requiresTeamScope ? watchedTeamId : undefined, + enabled: recommendationEnabled, + }; + const { + data: recommendation, + isPending: recommendationPending, + isError: recommendationFailed, + error: recommendationError, + refetch: refetchRecommendation, + } = useAutoRouterRecommendation(recommendationParams); + // A preset's models can only be trusted against a successfully loaded list. Selection and the // greyed-out state derive from this one function, so a preset that cannot be selected can never // have been applied: while loading we withhold selection rather than let a caller pick a preset @@ -302,7 +327,7 @@ const AddAutoRouterTab: React.FC = ({ [sortedPresetOptions], ); - const applyPrefill = (prefill: PresetPrefill) => { + const applyPrefill = React.useCallback((prefill: PresetPrefill) => { setEditingTiers(false); setComplexityRouterConfig(prefill.complexityRouterConfig); setCustomTechnicalKeywords(prefill.customTechnicalKeywords); @@ -311,7 +336,23 @@ const AddAutoRouterTab: React.FC = ({ setEmbeddingModel(prefill.embeddingModel); setMatchThreshold(prefill.matchThreshold); setEscalationKeywords(prefill.escalationKeywords); - }; + }, []); + + useEffect(() => { + if (!recommendation || setupMode !== "auto") return; + const key = [ + recommendation.snapshot_id, + qualityLevel, + optimizeFor, + watchedTeamId ?? "", + ...recommendation.matched_model_groups, + ].join(":"); + if (appliedRecommendationKey.current === key) return; + appliedRecommendationKey.current = key; + setSelectedPreset(undefined); + applyPrefill(buildPresetPrefill(recommendation.complexity_router_config, availability)); + setDetailsExpanded(false); + }, [recommendation, setupMode, qualityLevel, optimizeFor, watchedTeamId, availability, applyPrefill]); const handlePresetChange = (presetKey: string | undefined) => { if (!presetKey || presetKey === "custom") { @@ -342,13 +383,28 @@ const AddAutoRouterTab: React.FC = ({ defaultModel: complexityRouterConfig.default_model, }; - const submitBlockedReason = getSubmitBlockedReason( - complexityRouterConfig, - keywordTierRules, - referencedModelsParams, - groupsOnlyAvailability, - modelInfo, - ); + const recommendationStatus = { + setupMode, + recommendationEnabled, + recommendationPending, + recommendationFailed, + hasRecommendation: recommendation !== undefined, + }; + const recommendationBlockedReason = getRecommendationBlockedReason(recommendationStatus); + const regenerateAutoSetup = () => { + if (!recommendation) return; + applyPrefill(buildPresetPrefill(recommendation.complexity_router_config, availability)); + setDetailsExpanded(false); + }; + const submitBlockedReason = + recommendationBlockedReason ?? + getSubmitBlockedReason( + complexityRouterConfig, + keywordTierRules, + referencedModelsParams, + groupsOnlyAvailability, + modelInfo, + ); const complexityRouterConfigParams: BuildComplexityRouterConfigParams = { tiers: complexityRouterConfig.tiers, @@ -388,6 +444,7 @@ const AddAutoRouterTab: React.FC = ({ reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score, enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation, contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer, + autoSetup: complexityRouterConfig.auto_setup, }; const submitRecommendedRouter = async (name: string) => { @@ -395,13 +452,15 @@ const AddAutoRouterTab: React.FC = ({ // disagree about why. The handler needs it in its own right: the form fires this on Enter // regardless of the button's disabled state. const blockedReason = + recommendationBlockedReason ?? getSubmitBlockedReason( complexityRouterConfig, keywordTierRules, referencedModelsParams, groupsOnlyAvailability, modelInfo, - ) ?? getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); + ) ?? + getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); if (blockedReason) { setShowValidationErrors(true); toast.fromError(blockedReason); @@ -507,71 +566,6 @@ const AddAutoRouterTab: React.FC = ({ {({ ref, ...field }) => } -
- - - {modelsUnverifiable && ( -
- Could not load available models.{" "} - -
- )} - {presetsPending &&
Loading templates...
} - {presetsUnavailable && ( -
- Could not load templates, so only Custom Configuration is shown.{" "} - -
- )} -
- {requiresTeamScope && ( = ({ )} + {setupMode === "auto" ? ( + void refetchRecommendation()} + onRegenerate={regenerateAutoSetup} + onUseManual={() => setSetupMode("manual")} + /> + ) : ( +
+
+ + +
+ + {modelsUnverifiable && ( +
+ Could not load available models.{" "} + +
+ )} + {presetsPending &&
Loading templates...
} + {presetsUnavailable && ( +
+ Could not load templates, so only Custom Configuration is shown.{" "} + +
+ )} +
+ )} +