From e12415aac177610d539d354d02c2a07337130ec1 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Fri, 31 Jul 2026 22:50:07 -0700 Subject: [PATCH] fix(router): derive the savings baseline per call instead of caching it The parent router adds and removes deployments while it runs (`model_list` is appended to and popped from during deployment updates and health checks), so a baseline pinned on first access keeps naming a model the router no longer has, and a pricier deployment added later can never become the baseline. Nothing recomputes it, so the value stays wrong for the life of the instance. The cache was never worth having: resolving the baseline over a four-candidate router takes about 60 microseconds against a network call three orders of magnitude larger. Removing it also removes the question the previous round was spent answering, since there is no longer a cached value whose "not computed yet" state has to be told apart from "computed to nothing". A malformed `usage_object` now logs at warning rather than debug. It zeroes the auto-router driver for every affected row, and a shape change in `Usage` would otherwise surface only as a dashboard that quietly reads $0.00. --- litellm/proxy/spend_tracking/savings.py | 5 ++- .../auto_router/auto_router.py | 17 ++++---- .../router_strategy/test_auto_router.py | 40 ++++++++++++++----- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 00d6e82a79b..5eb6fc6cd9a 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -160,7 +160,10 @@ def _usage_from_spend_log(usage_object: dict | None) -> Usage | None: try: return Usage(**usage_object) except Exception as e: # noqa: BLE001 # a malformed usage_object must not fail the daily spend write - verbose_proxy_logger.debug("savings: unusable usage_object (%s)", e) + # Warning, not debug: this silently zeroes the auto-router driver for every + # affected row, and a shape change in Usage would otherwise show up only as a + # dashboard that quietly reads $0.00. + verbose_proxy_logger.warning("savings: unusable usage_object, auto-router savings will read zero (%s)", e) return None diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 7ef2b5a3562..cd074cd9097 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -2,7 +2,6 @@ Auto-Routing Strategy that works with a Semantic Router Config """ -from functools import cached_property from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from litellm._logging import verbose_router_logger @@ -135,14 +134,6 @@ class AutoRouter(CustomLogger): return None return max(priced)[2] - @cached_property - def _derived_savings_baseline_model(self) -> str | None: - """Resolved on first use, not at construction, because the parent router's - deployments are still being assembled while this router is built. Caching - through `cached_property` keeps "derived to nothing" distinct from "not derived - yet" without a second flag to hold them apart.""" - return self._most_expensive_candidate() - @property def savings_baseline_model(self) -> str | None: """The model this router's savings are measured against. @@ -158,13 +149,19 @@ class AutoRouter(CustomLogger): writes `deepseek-r1` meaning Azure would otherwise be priced against whoever owns that name. + Derived per call rather than cached: the parent router adds and removes + deployments while it runs, so a baseline pinned on first use would keep naming a + model the router no longer has, and a pricier one added later could never become + the baseline. Resolving costs tens of microseconds against a network call, which + is not worth trading correctness for. + ``None`` when nothing can be priced, which zeroes the driver rather than inventing a baseline. """ configured = self.configured_savings_baseline_model if configured: return self._canonical_model(configured, None) - return self._derived_savings_baseline_model + return self._most_expensive_candidate() def _load_semantic_routing_routes(self) -> List[Route]: from semantic_router.routers import SemanticRouter diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index ce8406a44e0..f4537e834cf 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -484,19 +484,39 @@ class TestSavingsBaselineModel: auto_router = self._auto_router({}, ["not-a-real-model"], "also-not-real") assert auto_router.savings_baseline_model is None - def test_the_derived_baseline_is_resolved_once(self): - """The parent router's deployments are still being assembled while this router - is constructed, so resolution is lazy; it must not re-derive per request.""" + def test_the_baseline_follows_deployments_added_after_the_first_read(self): + """The parent router adds and removes deployments while it runs. A baseline + pinned on first use would keep naming a model the router no longer has, and a + pricier one added later could never become the baseline.""" auto_router = self._auto_router( - {"cheap-tier": "anthropic/claude-haiku-4-5", "mid-tier": "anthropic/claude-sonnet-5"}, - ["cheap-tier", "mid-tier"], - "cheap-tier", + {"cheap": {"model": "claude-haiku-4-5", "custom_llm_provider": "anthropic"}}, + ["cheap", "big"], + "cheap", ) - assert auto_router.savings_baseline_model == "anthropic/claude-sonnet-5" + assert auto_router.savings_baseline_model == "anthropic/claude-haiku-4-5" - auto_router.litellm_router_instance.model_list = [] - auto_router.litellm_router_instance.model_name_to_deployment_indices = {} - assert auto_router.savings_baseline_model == "anthropic/claude-sonnet-5" + parent = auto_router.litellm_router_instance + parent.model_list.append({"model_name": "big", "litellm_params": {"model": "anthropic/claude-opus-5"}}) + parent.model_name_to_deployment_indices["big"] = [len(parent.model_list) - 1] + + assert auto_router.savings_baseline_model == "anthropic/claude-opus-5" + + def test_the_baseline_drops_a_deployment_that_was_removed(self): + auto_router = self._auto_router( + { + "cheap": {"model": "claude-haiku-4-5", "custom_llm_provider": "anthropic"}, + "big": {"model": "claude-opus-5", "custom_llm_provider": "anthropic"}, + }, + ["cheap", "big"], + "cheap", + ) + assert auto_router.savings_baseline_model == "anthropic/claude-opus-5" + + parent = auto_router.litellm_router_instance + parent.model_name_to_deployment_indices.pop("big") + auto_router.loaded_routes = [r for r in auto_router.loaded_routes if r.name != "big"] + + assert auto_router.savings_baseline_model == "anthropic/claude-haiku-4-5" def test_a_deployment_naming_its_provider_separately_is_still_priced(self): """A deployment may name its vendor in `custom_llm_provider` rather than in the