From aa59b88b6f9e96d7dc1e290759e81406dfdb7daf Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Wed, 29 Jul 2026 21:16:52 -0700 Subject: [PATCH] fix(complexity_router): never fall back a warming replay to an unvalidated group Eligibility, every-member cacheability, deployment affinity and pricing are properties of the target model group. When a replay failed and the key had not explicitly disabled fallbacks, the router could send it to a fallback group carrying none of that validation, spending the customer's money without warming the tier the session will actually switch to. Fallbacks are now disabled at the dispatch site rather than in the request body, so no key-level control or pre-call mutation can re-enable them. router.py:6157 raises before fallbacks, context_window_fallbacks and content_policy_fallbacks are consulted, so the single flag covers all three kinds; a failed replay simply retries on the next tick, which was already the documented intent. --- .../complexity_router/cache_warming/refresher.py | 9 ++++++++- .../complexity_router/cache_warming/test_refresher.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/complexity_router/cache_warming/refresher.py b/litellm/router_strategy/complexity_router/cache_warming/refresher.py index 6bb80233c4d..1161e422a75 100644 --- a/litellm/router_strategy/complexity_router/cache_warming/refresher.py +++ b/litellm/router_strategy/complexity_router/cache_warming/refresher.py @@ -99,7 +99,13 @@ def _replay_body( default would collide every replay onto one entry. The warming marker rides spend_logs_metadata, not metadata.tags: tags are an input to deployment selection (enable_tag_filtering makes an unmatched tag unroutable) and to policy (_reject_clientside_metadata_tags_check refuses any request carrying them), - while spend_logs_metadata exists to label spend rows, which is all this marker is for.""" + while spend_logs_metadata exists to label spend rows, which is all this marker is for. Fallbacks are disabled at the + dispatch site rather than in the body so no key-level or pre-call mutation can re-enable them: everything + warming validated -- prompt-cache eligibility, every-member cacheability, deployment affinity, pricing -- + is a property of the target GROUP, and a fallback substitutes a different group carrying none of it, + spending the customer's money to warm nothing. router.py:6157 raises before fallbacks, + context_window_fallbacks and content_policy_fallbacks are consulted, so one flag covers all three; a + failed replay simply retries next tick.""" from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup @@ -637,6 +643,7 @@ class CacheWarmingRefresher: if admitted is None: return False data, principal = admitted + data["disable_fallbacks"] = True try: await ( llm_router.aanthropic_messages(**data) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # factory-generated router surface is legacy-untyped diff --git a/tests/test_litellm/router_strategy/complexity_router/cache_warming/test_refresher.py b/tests/test_litellm/router_strategy/complexity_router/cache_warming/test_refresher.py index 1770eabd0dc..4fb1aa88524 100644 --- a/tests/test_litellm/router_strategy/complexity_router/cache_warming/test_refresher.py +++ b/tests/test_litellm/router_strategy/complexity_router/cache_warming/test_refresher.py @@ -370,3 +370,14 @@ async def test_a_direct_sdk_session_with_no_recorded_identity_still_warms_unattr await tick(llm_router, active=refresher(keys=FakeKeyDirectory({}))) assert replayed_models(llm_router) == ["smart-claude"] assert llm_router.completion_calls[0]["metadata"]["user_api_key_team_id"] is None + + +@pytest.mark.asyncio +async def test_a_replay_never_falls_back_to_a_group_warming_did_not_validate(): + """Eligibility, every-member cacheability, affinity and pricing are properties of the target GROUP, so a + fallback would substitute a group carrying none of them and spend the customer's money warming nothing.""" + llm_router, redis = warming_rig(redis=FakeRedisCache()) + seed_session(redis) + await tick(llm_router) + assert llm_router.completion_calls, "expected a replay" + assert all(call["disable_fallbacks"] is True for call in llm_router.completion_calls)