diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 66fd25e2e79..e1ac5ff6038 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -207,6 +207,7 @@ async def preview_auto_router_routing( litellm_router_instance=llm_router, complexity_router_config=data.complexity_router_config.model_dump(exclude_none=True), default_model=data.default_model, + derive_savings_baseline=False, ) request_kwargs: Final = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata( diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 7ac613dc7c6..a8fe023c802 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -298,6 +298,7 @@ def compute_autorouter_savings( usage: Usage, conversation_continuing: bool = True, selected_info: ModelInfo | None = None, + baseline_info: ModelInfo | None = None, cost_breakdown: Mapping[str, object] | None = None, ) -> float: """Net dollars the router saved, or cost, by serving this request on ``selected_model``. @@ -341,9 +342,12 @@ def compute_autorouter_savings( if baseline == selected: return 0.0 basis: Final = _pricing_basis(cost_breakdown) - baseline_info: Final = _model_info(baseline) + effective_baseline_info: Final = baseline_info if baseline_info is not None else _model_info(baseline) baseline_cost: Final = _cost_of_usage( - baseline, _baseline_usage(usage, conversation_continuing, baseline_info), baseline_info, basis + baseline, + _baseline_usage(usage, conversation_continuing, effective_baseline_info), + effective_baseline_info, + basis, ) # Falls back to pricing the request only when the biller recorded nothing, which is # every row written before the breakdown carried its basis. @@ -410,13 +414,14 @@ def compute_savings_spend( if usage is None or not model: return SavingsSpend(compression=compression, prompt_caching=prompt_caching) - # The counterfactual is one model an operator would have run instead of the router, - # configured once for the proxy rather than derived per request. Unset means the - # driver is off; a routing decision is what says this request was auto-routed at all. - # Both are checked before anything is resolved, because every spend write reaches - # here and only auto-routed ones can produce a number. - baseline_model: Final = litellm.autorouter_savings_baseline_model + # The configured `autorouter_savings_baseline_model` wins; otherwise the baseline + # the deciding router recorded on its decision; neither means the driver is off. decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {} + recorded: Final = decision.get("savings_baseline_model") + recorded_id: Final = decision.get("savings_baseline_deployment_id") + configured: Final = litellm.autorouter_savings_baseline_model + baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None) + baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None autorouter: Final = ( compute_autorouter_savings( baseline_model=baseline_model, @@ -426,7 +431,10 @@ def compute_savings_spend( # Absent means the router never recorded a shape, which is the conservative # reading: charge the cache write rather than claim a first turn's saving. conversation_continuing=decision.get("conversation_continuing") is not False, - selected_info=_effective_model_info(llm_router() if llm_router else None, model_id, model or ""), + selected_info=_effective_model_info( + (router_instance := llm_router() if llm_router else None), model_id, model or "" + ), + baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""), cost_breakdown=cost_breakdown, ) if decision and baseline_model diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index ea2cc7297dd..00be376bcc6 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -52,6 +52,7 @@ if TYPE_CHECKING: from litellm.router import Router from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter + from litellm.router_strategy.savings_baseline import Baseline from litellm.types.router import PreRoutingHookResponse else: Router = Any @@ -417,6 +418,7 @@ class ComplexityRouter(CustomLogger): litellm_router_instance: Router, complexity_router_config: dict[str, Any] | None = None, default_model: str | None = None, + derive_savings_baseline: bool = True, ): """ Initialize ComplexityRouter. @@ -426,9 +428,13 @@ class ComplexityRouter(CustomLogger): litellm_router_instance: The LiteLLM Router instance. complexity_router_config: Optional configuration dict from proxy config. default_model: Optional default model to use if tier cannot be determined. + derive_savings_baseline: False for callers whose decisions are never spend + tracked, such as the routing-test preview, where the resolved baseline + would leak deployment mappings the caller was not authorized for. """ self.model_name = model_name self.litellm_router_instance = litellm_router_instance + self._derive_savings_baseline = derive_savings_baseline # Parse config - always create a new instance to avoid singleton mutation if complexity_router_config: @@ -474,9 +480,45 @@ class ComplexityRouter(CustomLogger): self.adaptive_router: AdaptiveRouter | None = None self._model_tiers: dict[str, tuple[ComplexityTier, ...]] = {} self._adaptive_init_attempted = False + self._savings_baseline: Baseline | None = None + self._savings_baseline_derived = False verbose_router_logger.debug("ComplexityRouter initialized for %s with tiers: %s", model_name, self.config.tiers) + def _hardest_tier_models(self) -> tuple[str, ...]: + """The model pool of the most severe tier this router configures. + + The hardest *configured* tier, not REASONING unconditionally: a deployment + that only defines SIMPLE and MEDIUM is still measured against the best it + could actually have picked. + """ + for tier in reversed(TIER_SEVERITY_ORDER): + models = self.config.tiers.get(tier.value) + if models: + return tuple(models) if isinstance(models, list) else (models,) + return () + + @property + def savings_baseline(self) -> Baseline | None: + """The derived counterfactual this router's savings are measured against. + + ``None`` when `litellm_settings.autorouter_savings_baseline_model` is set (the + spend writer reads that setting directly and it wins) or when this router was + built with ``derive_savings_baseline=False``. Derived once on first use and + pinned for the instance's lifetime: creating or editing the router rebuilds + the instance, which re-derives. Deferred past ``__init__`` because during a + config load this router can be constructed before its tier deployments are. + """ + import litellm + from litellm.router_strategy.savings_baseline import resolve_baseline + + if not self._derive_savings_baseline or litellm.autorouter_savings_baseline_model is not None: + return None + if not self._savings_baseline_derived: + self._savings_baseline = resolve_baseline(self.litellm_router_instance, self._hardest_tier_models()) + self._savings_baseline_derived = True + return self._savings_baseline + def _estimate_tokens(self, text: str) -> int: """ Estimate token count from text. @@ -716,6 +758,10 @@ class ComplexityRouter(CustomLogger): cause=cause, conversation_continuing=conversation_continuing, ) + if (baseline := self.savings_baseline) is not None: + decision["savings_baseline_model"] = baseline.model + if baseline.deployment_id is not None: + decision["savings_baseline_deployment_id"] = baseline.deployment_id if tier is not None: decision["tier"] = tier.value if score is not None: diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py new file mode 100644 index 00000000000..e10ec4a1e6f --- /dev/null +++ b/litellm/router_strategy/savings_baseline.py @@ -0,0 +1,157 @@ +"""The default counterfactual a complexity router's savings are measured against. + +`litellm_settings.autorouter_savings_baseline_model` names the model the traffic would +have run on without a router. When the operator sets it, that answer wins and nothing +here runs. When they do not, the router's own tier ladder already names it: without a +router a deployment has to pick one model that can carry the hardest request it will +see, so the default baseline is the priciest model in the hardest configured tier. A +cheap tier is a choice the router made, not a ceiling it was bounded by. + +Candidates are ranked once against a fixed reference request, not against each request +that runs. Ranking per request means reading the request, and every input shape it can +take; a default must not carry that surface. An operator whose pool ordering genuinely +depends on request shape names the baseline in config, which skips this file entirely. + +Baselines are always provider-qualified, because they travel to the spend writer as a +bare string with no provider beside them; an operator who writes ``deepseek-r1`` meaning +Azure would otherwise be priced against whoever else owns that name. +""" + +from collections.abc import Iterable +from typing import TYPE_CHECKING, Final, NamedTuple + +from litellm._logging import verbose_router_logger +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + +if TYPE_CHECKING: + from litellm.router import Router + + +_REFERENCE_REQUEST: Final = Usage( + prompt_tokens=20_000, + completion_tokens=1_000, + total_tokens=21_000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=19_000, cache_creation_tokens=1_000, text_tokens=0), +) + + +class Baseline(NamedTuple): + """The counterfactual deployment: what it is called, and which deployment it was. + + ``model`` is what the operator would recognise, and the string the spend writer + prices the counterfactual under. ``deployment_id`` only ranks: a deployment can be + charged something other than its model's public rate, and + `Router.get_deployment_model_info` is what merges the two. + """ + + model: str + deployment_id: str | None = None + + +def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None: + """``provider/model``, or ``None`` when the pair names no known provider. + + A deployment may name its vendor in the model prefix or in a separate + ``custom_llm_provider``, and the bare name alone is not enough to price: it can + resolve to a different vendor's rates, or to nothing at all. + """ + import litellm + + try: + resolved, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception as e: # noqa: BLE001 # an unroutable candidate cannot be the baseline + verbose_router_logger.debug("savings baseline: cannot resolve candidate %s (%s)", model, e) + return None + return f"{provider}/{resolved}" + + +def _models_in(router: "Router", group_name: str) -> tuple[Baseline, ...]: + """The candidates a tier entry actually calls, each with its own pricing key. + + `litellm_params.model` is not always a model: on Azure it is the deployment name, + absent from the cost map, and `model_info.base_model` names the real one. Wildcard + and aliased deployments behave the same, and router.py resolves pricing through + that same base_model chain. A name matching no deployment is a tier pointing + straight at a provider model rather than at a configured group, and prices under + its own name because there is no deployment to override it. + """ + indices: Final = router.model_name_to_deployment_indices.get(group_name) + if not indices: + return (Baseline(qualified),) if (qualified := canonical_model(group_name)) else () + + def candidate(index: int) -> Baseline | None: + deployment: Final = router.model_list[index] + params: Final = deployment.get("litellm_params") + if not isinstance(params, dict): + return None + info: Final = deployment.get("model_info") + base: Final = info.get("base_model") if isinstance(info, dict) else None + model: Final = base or params.get("base_model") or params.get("model") + qualified: Final = canonical_model(model, params.get("custom_llm_provider")) if model else None + if qualified is None: + return None + deployment_id: Final = info.get("id") if isinstance(info, dict) else None + return Baseline(qualified, str(deployment_id) if deployment_id else None) + + return tuple(c for index in indices if (c := candidate(index)) is not None) + + +def _priced(router: "Router", candidate: Baseline) -> tuple[float, Baseline] | None: + """``(cost_of_the_reference_request, candidate)``, or ``None`` when unpriceable. + + "Most expensive" is a property of a request, not of a rate: a deployment dearer per + output token can be cheaper per cached token, so comparing a chosen pair of rates + orders cache-heavy traffic backwards. Costing one reference request through the same + engine the savings use leaves cache rates, tiered tables and every other billing + dimension to that engine. A candidate that prices to nothing there cannot stand in + for what the traffic would have cost. + """ + from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token + + provider, _, model_name = candidate.model.partition("/") + try: + info: Final = router.get_deployment_model_info(candidate.deployment_id or "", candidate.model) + if info is None: + return None + prompt_cost, completion_cost = generic_cost_per_token( + model=model_name or candidate.model, + usage=_REFERENCE_REQUEST, + custom_llm_provider=provider, + model_info=info, + ) + except Exception as e: # noqa: BLE001 # an unpriceable candidate simply cannot be the baseline + verbose_router_logger.debug("savings baseline: no pricing for candidate %s (%s)", candidate.model, e) + return None + cost: Final = prompt_cost + completion_cost + if cost <= 0.0: + verbose_router_logger.debug("savings baseline: candidate %s prices to nothing", candidate.model) + return None + return (cost, candidate) + + +def _most_expensive(router: "Router", candidates: Iterable[Baseline]) -> Baseline | None: + """The candidate that would have cost the most on the reference request.""" + priced: Final = tuple(r for candidate in candidates if (r := _priced(router, candidate)) is not None) + if not priced: + verbose_router_logger.debug("savings baseline: no priceable candidates; savings driver disabled") + return None + return max(priced)[1] + + +def resolve_baseline(router: "Router", group_names: Iterable[str]) -> Baseline | None: + """The derived baseline for a router whose hardest tier offers ``group_names``. + + Holds no cache of its own; each pricing pass walks the pool, so the caller is + expected to bound how often it runs. The complexity router caches the result with a + TTL, which keeps a deployment added or removed at runtime able to change the + baseline while keeping this walk off the per-request hot path. + + Never raises. This is read on the routing path to decorate a request that is about + to be served, and a dashboard's counterfactual is not worth failing a live request + over; an unresolvable baseline zeroes the savings driver instead. + """ + try: + return _most_expensive(router, (c for name in group_names for c in _models_in(router, name))) + except Exception as e: # noqa: BLE001 # see docstring: routing must not fail for a metric + verbose_router_logger.warning("savings baseline: could not resolve, savings will read zero (%s)", e) + return None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5198008687f..77f83c5b6f8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2807,6 +2807,8 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): escalated: bool tier_boundaries: StandardLoggingRoutingDecisionTierBoundaries conversation_continuing: bool + savings_baseline_model: str + savings_baseline_deployment_id: str # Fields whose values quote the caller's prompt. Dropped when an operator turns message @@ -2827,6 +2829,8 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[FrozenSet[str]] = frozenset( "escalated", "tier_boundaries", "conversation_continuing", + "savings_baseline_model", + "savings_baseline_deployment_id", } ) diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index 1d6b0da561d..bd5551c4d89 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -485,6 +485,7 @@ def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write(): ) assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation" + def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt(): """OpenAI, Azure and Gemini entries carry no `cache_creation_input_token_cost`, because those providers cache implicitly and charge nothing to write. Leaving this @@ -505,9 +506,7 @@ def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt(): assert gpt5.get("cache_creation_input_token_cost") is None, "pick a baseline with no cache-write rate" haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") baseline_pays_input = 20_000 * gpt5["input_cost_per_token"] + 1_000 * gpt5["output_cost_per_token"] - actually_paid = ( - 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] - ) + actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] assert reported == pytest.approx(baseline_pays_input - actually_paid) assert reported > 0, "routing a cold first turn onto a cheaper model is a saving, not a loss" @@ -530,9 +529,7 @@ def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate(): assert grok.get("cache_read_input_token_cost") is None, "pick a baseline with no cache-read rate" haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic") baseline_pays_input = 20_000 * grok["input_cost_per_token"] + 1_000 * grok["output_cost_per_token"] - actually_paid = ( - 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] - ) + actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"] assert reported == pytest.approx(baseline_pays_input - actually_paid) @@ -617,3 +614,101 @@ def test_the_baseline_is_priced_on_the_basis_the_request_was_billed_at(basis, ex baseline = 20_000 * gpt["input_cost_per_token"] + 1_000 * gpt["output_cost_per_token"] assert reported == pytest.approx(expected_multiplier * baseline - served) + + +def test_a_baseline_recorded_on_the_decision_turns_the_driver_on(): + """An operator who configures nothing still sees the driver work.""" + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5"}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter != 0.0 + + +def test_the_configured_baseline_overrides_the_recorded_one(monkeypatch): + """The recorded baseline and its deployment id are both ignored under the setting.""" + monkeypatch.setattr(litellm, "autorouter_savings_baseline_model", "claude-sonnet-5") + with_override = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision={ + "conversation_continuing": True, + "savings_baseline_model": "anthropic/claude-opus-5", + "savings_baseline_deployment_id": "some-deployment-id", + }, + usage_object=_cached_usage_object(), + ) + against_sonnet = compute_autorouter_savings( + baseline_model="claude-sonnet-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=Usage(**_cached_usage_object()), + ) + against_opus = compute_autorouter_savings( + baseline_model="anthropic/claude-opus-5", + selected_model="claude-haiku-4-5", + selected_provider="anthropic", + usage=Usage(**_cached_usage_object()), + ) + assert against_sonnet != against_opus, "the test needs baselines that price apart" + assert with_override.autorouter == against_sonnet + + +def test_a_non_string_recorded_baseline_is_ignored(): + result = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision={"conversation_continuing": True, "savings_baseline_model": ["anthropic/claude-opus-5"]}, + usage_object=_cached_usage_object(), + ) + assert result.autorouter == 0.0 + + +def test_a_recorded_baseline_deployment_prices_at_its_configured_rate(): + """A hardest-tier deployment with a negotiated rate is what the traffic would + really have cost; pricing its model publicly misstates the saving.""" + router = Router( + model_list=[ + { + "model_name": "top", + "litellm_params": { + "model": "anthropic/claude-opus-5", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + }, + ] + ) + deployment_id = router.get_model_list(model_name="top")[0]["model_info"]["id"] + decision = { + "conversation_continuing": True, + "savings_baseline_model": "anthropic/claude-opus-5", + "savings_baseline_deployment_id": deployment_id, + } + with_deployment_rate = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision=decision, + usage_object=_cached_usage_object(), + llm_router=lambda: router, + ) + at_public_rate = compute_savings_spend( + model="claude-haiku-4-5", + custom_llm_provider="anthropic", + compression_saved_tokens=0, + cache_read_input_tokens=0, + routing_decision={k: v for k, v in decision.items() if k != "savings_baseline_deployment_id"}, + usage_object=_cached_usage_object(), + llm_router=lambda: router, + ) + assert with_deployment_rate.autorouter > at_public_rate.autorouter diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index d713fc9e0f4..f7ca6090bb1 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -746,7 +746,7 @@ class TestSingletonMutation: def test_default_config_not_mutated(self, mock_router_instance): """Test that creating routers without config doesn't mutate defaults.""" from litellm.router_strategy.complexity_router.config import ( - DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, + DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE, ComplexityRouterConfig, ) @@ -4050,18 +4050,14 @@ class TestRoutingDecisionIsPerAttempt: {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}}, ] - @pytest.mark.parametrize( - "seed, bucket", [({}, "metadata"), ({"litellm_metadata": {}}, "litellm_metadata")] - ) + @pytest.mark.parametrize("seed, bucket", [({}, "metadata"), ({"litellm_metadata": {}}, "litellm_metadata")]) @pytest.mark.asyncio async def test_fallback_to_plain_model_group_clears_the_earlier_decision(self, seed, bucket): router = Router(model_list=self.MODEL_LIST) request_kwargs: Dict = dict(seed) messages = [{"role": "user", "content": "Hello!"}] - await router.async_pre_routing_hook( - model="smart-router", request_kwargs=request_kwargs, messages=messages - ) + await router.async_pre_routing_hook(model="smart-router", request_kwargs=request_kwargs, messages=messages) assert "routing_decision" in request_kwargs[bucket] # The fallback attempt reuses the same kwargs and selects no strategy. @@ -4113,7 +4109,6 @@ class TestRecordRoutingDecision: Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) assert request_kwargs == {} - def test_clearing_the_decision_takes_the_savings_facts_with_it(self): """A fallback to a plain model group re-enters the hook with the same `request_kwargs`. The baseline and the conversation shape ride inside the @@ -4393,7 +4388,12 @@ class TestContextAwareClassifier: id="multiple-reminders-stripped", ), pytest.param( - [{"role": "user", "content": [{"type": "text", "text": _REMINDER}, {"type": "text", "text": "and now?"}]}], + [ + { + "role": "user", + "content": [{"type": "text", "text": _REMINDER}, {"type": "text", "text": "and now?"}], + } + ], "and now?", id="reminder-in-its-own-content-part", ), @@ -4780,9 +4780,7 @@ class TestContextAwareClassifier: assert reported > 100 @pytest.mark.asyncio - async def test_no_trajectory_signal_when_request_had_no_messages( - self, llm_complexity_router, mock_router_instance - ): + async def test_no_trajectory_signal_when_request_had_no_messages(self, llm_complexity_router, mock_router_instance): """On the prompt-only path there is no conversation to measure, so the depth line is omitted rather than asserting a false "~0 tokens" to the classifier.""" mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "SIMPLE"}')) @@ -4794,9 +4792,7 @@ class TestContextAwareClassifier: assert "what is 2+2" in user_payload @pytest.mark.asyncio - async def test_single_turn_request_sends_no_conversation_context( - self, llm_complexity_router, mock_router_instance - ): + async def test_single_turn_request_sends_no_conversation_context(self, llm_complexity_router, mock_router_instance): """A single-turn request carries no conversation, so the classifier sees only the ask. Found in QA: the depth line gated on `messages` being non-empty, so single-turn requests got a @@ -4846,7 +4842,6 @@ class TestContextAwareClassifier: assert "sharding strategy" not in user_payload assert user_payload.strip() == "Classify this message:\nwhat is 2+2" - @pytest.mark.asyncio @pytest.mark.parametrize("include_assistant,plan_is_quoted", [(True, True), (False, False)]) async def test_assistant_turn_carrying_the_difficulty_reaches_the_classifier( @@ -4888,7 +4883,6 @@ class TestContextAwareClassifier: assert (f"[1] {ask}" in user_payload) is not plan_is_quoted assert user_payload.endswith("Classify this message:\nyes.") - @pytest.mark.asyncio @pytest.mark.parametrize("include_assistant", [True, False]) async def test_depth_signal_agrees_with_what_the_window_quoted( @@ -5010,9 +5004,6 @@ class TestClassifierTrustBoundary: assert hostile not in system_message["content"] assert hostile in user_message["content"] - - - @pytest.mark.parametrize( "window_size,conversation_is_quoted", [ @@ -5039,7 +5030,6 @@ class TestClassifierTrustBoundary: assert ('short reply such as "yes" or "continue"' in system_prompt) is conversation_is_quoted assert ("Classify only the current message" in system_prompt) is not conversation_is_quoted - @pytest.mark.asyncio @pytest.mark.parametrize("include_assistant", [True, False]) async def test_context_framing_does_not_depend_on_which_roles_the_window_holds( @@ -5209,7 +5199,10 @@ class TestConversationShapeDiscriminator: def test_a_system_prompt_does_not_make_a_first_turn_look_continued(self): from litellm.router_strategy.complexity_router.complexity_router import _conversation_is_continuing - assert _conversation_is_continuing([{"role": "system", "content": "s"}, {"role": "user", "content": "hi"}]) is False + assert ( + _conversation_is_continuing([{"role": "system", "content": "s"}, {"role": "user", "content": "hi"}]) + is False + ) def test_unreadable_messages_stay_conservative(self): """No messages says nothing about the baseline's cache, so it keeps charging the @@ -5233,5 +5226,133 @@ class TestConversationShapeDiscriminator: ) builds = source.split("self._build_routing_decision(")[1:] assert builds - missing = [i for i, block in enumerate(builds) if "conversation_continuing=conversation_continuing" not in block.split("),")[0]] + missing = [ + i + for i, block in enumerate(builds) + if "conversation_continuing=conversation_continuing" not in block.split("),")[0] + ] assert not missing, f"routing decisions {missing} do not carry the conversation shape" + + +class TestSavingsBaselineOnDecision: + """The derived counterfactual rides on every routing decision, recorded by the + deciding instance because tag-scoped routers under one model name make a + spend-write-time lookup ambiguous.""" + + @staticmethod + def _router_with_tiers(tiers: dict, **kwargs) -> ComplexityRouter: + parent = Router( + model_list=[ + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "mid", "litellm_params": {"model": "anthropic/claude-sonnet-5"}}, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-fable-5"}}, + ] + ) + return ComplexityRouter( + model_name="savings-router", + litellm_router_instance=parent, + complexity_router_config={"tiers": tiers}, + **kwargs, + ) + + def test_derives_the_priciest_model_of_the_reasoning_tier(self): + router = self._router_with_tiers({"SIMPLE": "cheap", "MEDIUM": "mid", "REASONING": ["cheap", "top"]}) + assert router.savings_baseline.model == "anthropic/claude-fable-5" + + def test_falls_back_to_the_hardest_configured_tier_when_reasoning_is_absent(self): + """A router defining only SIMPLE and MEDIUM is measured against the best it + could actually have picked, not a tier it never had.""" + router = self._router_with_tiers({"SIMPLE": "cheap", "MEDIUM": "mid"}) + assert router.savings_baseline.model == "anthropic/claude-sonnet-5" + + def test_a_configured_proxy_wide_baseline_disables_derivation(self, monkeypatch): + monkeypatch.setattr(litellm, "autorouter_savings_baseline_model", "claude-opus-5") + router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": "top"}) + assert router.savings_baseline is None + + def test_the_decision_record_carries_the_derived_baseline_and_its_deployment(self): + """The deployment id is what lets the spend writer price a baseline whose + deployment carries a configured rate instead of the public one.""" + router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": "top"}) + expected_id = router.litellm_router_instance.get_model_list(model_name="top")[0]["model_info"]["id"] + decision = router._build_routing_decision(routed_model="cheap", cause="heuristic_scorer") + assert decision["savings_baseline_model"] == "anthropic/claude-fable-5" + assert decision["savings_baseline_deployment_id"] == expected_id + + def test_an_unresolvable_baseline_is_omitted_not_recorded_as_none(self): + router = self._router_with_tiers({"SIMPLE": "utter-nonsense-no-provider-owns"}) + decision = router._build_routing_decision(routed_model="cheap", cause="heuristic_scorer") + assert "savings_baseline_model" not in decision + assert "savings_baseline_deployment_id" not in decision + + def test_a_router_built_without_derivation_records_nothing(self): + """The routing-test preview returns the decision verbatim to callers who are + only authorized for the classifier and embedding models, so its router must + not resolve tier groups into deployment mappings.""" + router = self._router_with_tiers({"SIMPLE": "cheap", "REASONING": "top"}, derive_savings_baseline=False) + assert router.savings_baseline is None + decision = router._build_routing_decision(routed_model="cheap", cause="heuristic_scorer") + assert "savings_baseline_model" not in decision + assert "savings_baseline_deployment_id" not in decision + + def test_the_routing_test_preview_builds_its_router_without_derivation(self): + import inspect + + from litellm.proxy.management_endpoints import auto_router_endpoints + + source = inspect.getsource(auto_router_endpoints.preview_auto_router_routing) + assert "derive_savings_baseline=False" in source + + +class TestSavingsBaselinePinnedPerInstance: + """Derivation walks and prices the hardest tier's pool, so it runs once per router + instance; the create and edit flows rebuild the instance, which re-derives.""" + + @staticmethod + def _router_and_parent() -> tuple[ComplexityRouter, Router]: + parent = Router( + model_list=[ + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-sonnet-5"}}, + ] + ) + router = ComplexityRouter( + model_name="savings-router", + litellm_router_instance=parent, + complexity_router_config={"tiers": {"SIMPLE": "cheap", "REASONING": ["cheap", "top"]}}, + ) + return router, parent + + def test_the_first_derivation_is_pinned_for_the_instance_lifetime(self): + router, parent = self._router_and_parent() + assert router.savings_baseline.model == "anthropic/claude-sonnet-5" + parent.model_name_to_deployment_indices.clear() + assert router.savings_baseline.model == "anthropic/claude-sonnet-5" + + def test_a_rebuilt_instance_re_derives_from_the_live_router(self): + """Editing a router goes through unregister and re-add, so a fresh instance is + what carries a config change into the baseline.""" + router, parent = self._router_and_parent() + assert router.savings_baseline.model == "anthropic/claude-sonnet-5" + parent.model_name_to_deployment_indices.clear() + rebuilt = ComplexityRouter( + model_name="savings-router", + litellm_router_instance=parent, + complexity_router_config={"tiers": {"SIMPLE": "cheap", "REASONING": ["cheap", "top"]}}, + ) + assert rebuilt.savings_baseline is None + + def test_the_configured_setting_bypasses_the_pin(self, monkeypatch): + router, _ = self._router_and_parent() + assert router.savings_baseline.model == "anthropic/claude-sonnet-5" + monkeypatch.setattr(litellm, "autorouter_savings_baseline_model", "claude-opus-5") + assert router.savings_baseline is None + + def test_an_unresolvable_pool_is_derived_once_and_pinned_as_none(self): + router, parent = self._router_and_parent() + parent.model_name_to_deployment_indices.clear() + router.config.tiers = {"SIMPLE": "utter-nonsense-no-provider-owns"} + assert router.savings_baseline is None + assert router._savings_baseline_derived is True + router.config.tiers = {"SIMPLE": "claude-haiku-4-5"} + assert router.savings_baseline is None diff --git a/tests/test_litellm/router_strategy/test_savings_baseline.py b/tests/test_litellm/router_strategy/test_savings_baseline.py new file mode 100644 index 00000000000..0766083aed5 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_savings_baseline.py @@ -0,0 +1,186 @@ +import pytest + +from litellm.router import Router +from litellm.router_strategy.savings_baseline import ( + Baseline, + canonical_model, + _models_in, + _most_expensive, + resolve_baseline, +) + + +@pytest.fixture +def parent() -> Router: + return Router( + model_list=[ + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-5"}}, + {"model_name": "pool", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "pool", "litellm_params": {"model": "anthropic/claude-opus-5"}}, + ] + ) + + +class TestCanonicalModel: + def test_qualifies_a_bare_name_with_the_provider_that_owns_it(self): + assert canonical_model("claude-opus-5") == "anthropic/claude-opus-5" + + def test_keeps_an_already_qualified_name_qualified(self): + assert canonical_model("anthropic/claude-opus-5") == "anthropic/claude-opus-5" + + def test_honours_a_separately_declared_provider(self): + assert canonical_model("claude-opus-5", "openai") == "openai/claude-opus-5" + + def test_returns_none_for_a_name_no_provider_claims(self): + assert canonical_model("") is None + + +class TestModelsForGroup: + def test_resolves_a_group_to_the_models_its_deployments_call(self, parent): + assert [c.model for c in _models_in(parent, "cheap")] == ["anthropic/claude-haiku-4-5"] + + def test_returns_every_deployment_in_a_pooled_group(self, parent): + assert sorted(c.model for c in _models_in(parent, "pool")) == [ + "anthropic/claude-haiku-4-5", + "anthropic/claude-opus-5", + ] + + def test_treats_an_unknown_group_as_a_model_name(self, parent): + """A tier can point straight at a provider model rather than a configured group.""" + assert [c.model for c in _models_in(parent, "claude-opus-5")] == ["anthropic/claude-opus-5"] + + +class TestMostExpensive: + """Ranking runs through the router, because what a deployment costs is the + router's answer to give: it merges configured prices over the built-in map.""" + + def test_picks_by_output_rate(self, parent): + picked = _most_expensive(parent, [Baseline("anthropic/claude-haiku-4-5"), Baseline("anthropic/claude-opus-5")]) + assert picked.model == "anthropic/claude-opus-5" + + def test_ignores_models_with_no_per_token_price(self, parent): + """A free model as baseline would report the whole real spend as a loss.""" + picked = _most_expensive( + parent, [Baseline("not-a-real-model-anywhere"), Baseline("anthropic/claude-haiku-4-5")] + ) + assert picked.model == "anthropic/claude-haiku-4-5" + + def test_returns_none_when_nothing_can_be_priced(self, parent): + assert _most_expensive(parent, [Baseline("not-a-real-model-anywhere")]) is None + + def test_returns_none_for_an_empty_candidate_set(self, parent): + assert _most_expensive(parent, []) is None + + +class TestResolveBaseline: + def test_derives_the_priciest_candidate(self, parent): + assert resolve_baseline(parent, ["cheap", "top"]).model == "anthropic/claude-opus-5" + + def test_never_raises_so_a_metric_cannot_fail_a_live_request(self): + """Read on the routing path while decorating a request that is about to be + served; a dashboard counterfactual must not be able to take routing down.""" + + class Exploding: + @property + def model_name_to_deployment_indices(self): + raise RuntimeError("router is mid-reload") + + assert resolve_baseline(Exploding(), ["anything"]) is None + + def test_an_empty_candidate_set_zeroes_the_driver_rather_than_inventing_one(self, parent): + assert resolve_baseline(parent, []) is None + + +class TestDeploymentsPricedByBaseModel: + """`litellm_params.model` is not always a model. + + On Azure it is the deployment name, which is absent from the cost map, so pricing it + directly drops the candidate. If that candidate was the priciest, the baseline quietly + becomes the second priciest and every saving is understated; if the whole pool is + Azure, nothing prices and the driver reports zero with nothing at default log level + saying why. `model_info.base_model` is what names the real model, which is the chain + router.py already resolves pricing through. + """ + + @staticmethod + def _router(*deployments: dict) -> Router: + return Router(model_list=list(deployments)) + + def test_model_info_base_model_is_preferred_over_the_deployment_name(self): + router = self._router( + { + "model_name": "big", + "litellm_params": {"model": "azure/my-gpt5-deployment"}, + "model_info": {"base_model": "azure/gpt-4.1"}, + }, + ) + assert [c.model for c in _models_in(router, "big")] == ["azure/gpt-4.1"] + + def test_litellm_params_base_model_is_the_other_accepted_spelling(self): + router = self._router( + { + "model_name": "big", + "litellm_params": {"model": "azure/my-gpt5-deployment", "base_model": "azure/gpt-4.1"}, + }, + ) + assert [c.model for c in _models_in(router, "big")] == ["azure/gpt-4.1"] + + def test_a_deployment_without_a_base_model_still_prices_by_its_model(self): + router = self._router({"model_name": "big", "litellm_params": {"model": "anthropic/claude-opus-5"}}) + assert [c.model for c in _models_in(router, "big")] == ["anthropic/claude-opus-5"] + + def test_an_azure_deployment_can_win_the_priciest_candidate(self): + """Without the base_model hop the Azure candidate never prices, so the cheaper + model wins by default and the reported saving shrinks.""" + router = self._router( + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + { + "model_name": "big", + "litellm_params": {"model": "azure/my-gpt5-deployment"}, + "model_info": {"base_model": "azure/gpt-4.1"}, + }, + ) + assert resolve_baseline(router, ["cheap", "big"]).model == "azure/gpt-4.1" + + def test_an_all_azure_pool_still_has_a_baseline(self): + """Otherwise nothing prices, the driver is disabled and the card reads $0.00.""" + router = self._router( + { + "model_name": "big", + "litellm_params": {"model": "azure/my-gpt5-deployment"}, + "model_info": {"base_model": "azure/gpt-4.1"}, + }, + ) + assert resolve_baseline(router, ["big"]).model == "azure/gpt-4.1" + + +class TestDeploymentPricingOverrides: + """A deployment may not be charged the public rate for the model it names.""" + + def test_a_configured_price_decides_the_baseline_not_the_public_rate(self): + """A deployment configured far above its public rate is what the traffic would + really have cost. Ranking on the public rate picks the wrong counterfactual and + then prices it at a rate nobody pays.""" + router = Router( + model_list=[ + {"model_name": "cheap", "litellm_params": {"model": "anthropic/claude-haiku-4-5"}}, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-5"}}, + ] + ) + assert resolve_baseline(router, ["cheap", "top"]).model == "anthropic/claude-opus-5" + + overridden = Router( + model_list=[ + { + "model_name": "cheap", + "litellm_params": { + "model": "anthropic/claude-haiku-4-5", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + }, + {"model_name": "top", "litellm_params": {"model": "anthropic/claude-opus-5"}}, + ] + ) + assert resolve_baseline(overridden, ["cheap", "top"]).model == "anthropic/claude-haiku-4-5" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 864ef80d39d..df85decc676 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31329,6 +31329,14 @@ export interface components { * @description Keywords indicating reasoning-required content */ reasoning_keywords?: string[] | null; + /** + * Reminder Markers + * @description Override the (open, close) marker pair used to recognize and strip harness-injected reminder blocks before classification. Defaults to Claude Code's convention, ('', ''), when unset. Matching is case-insensitive. + */ + reminder_markers?: [ + string, + string + ] | null; /** * Return Raw Model Name * @description Return the resolved raw model name in the response model field instead of the client-requested complexity-router alias @@ -32147,6 +32155,10 @@ export interface components { * @enum {string} */ router_type?: "complexity" | "adaptive" | "quality"; + /** Savings Baseline Deployment Id */ + savings_baseline_deployment_id?: string; + /** Savings Baseline Model */ + savings_baseline_model?: string; /** Score */ score?: number; /** Signals */