mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(complexity_router): never fall back a warming replay to an unvalidated group
Eligibility, every-member cacheability, deployment affinity and pricing are properties of the target model group. When a replay failed and the key had not explicitly disabled fallbacks, the router could send it to a fallback group carrying none of that validation, spending the customer's money without warming the tier the session will actually switch to. Fallbacks are now disabled at the dispatch site rather than in the request body, so no key-level control or pre-call mutation can re-enable them. router.py:6157 raises before fallbacks, context_window_fallbacks and content_policy_fallbacks are consulted, so the single flag covers all three kinds; a failed replay simply retries on the next tick, which was already the documented intent.
This commit is contained in:
parent
22635302bd
commit
343201e4b9
2 changed files with 19 additions and 1 deletions
|
|
@ -99,7 +99,13 @@ def _replay_body(
|
|||
default would collide every replay onto one entry. The warming marker rides spend_logs_metadata, not
|
||||
metadata.tags: tags are an input to deployment selection (enable_tag_filtering makes an unmatched tag
|
||||
unroutable) and to policy (_reject_clientside_metadata_tags_check refuses any request carrying them),
|
||||
while spend_logs_metadata exists to label spend rows, which is all this marker is for."""
|
||||
while spend_logs_metadata exists to label spend rows, which is all this marker is for. Fallbacks are disabled at the
|
||||
dispatch site rather than in the body so no key-level or pre-call mutation can re-enable them: everything
|
||||
warming validated -- prompt-cache eligibility, every-member cacheability, deployment affinity, pricing --
|
||||
is a property of the target GROUP, and a fallback substitutes a different group carrying none of it,
|
||||
spending the customer's money to warm nothing. router.py:6157 raises before fallbacks,
|
||||
context_window_fallbacks and content_policy_fallbacks are consulted, so one flag covers all three; a
|
||||
failed replay simply retries next tick."""
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
|
||||
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
||||
|
||||
|
|
@ -637,6 +643,7 @@ class CacheWarmingRefresher:
|
|||
if admitted is None:
|
||||
return False
|
||||
data, principal = admitted
|
||||
data["disable_fallbacks"] = True
|
||||
try:
|
||||
await (
|
||||
llm_router.aanthropic_messages(**data) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # factory-generated router surface is legacy-untyped
|
||||
|
|
|
|||
|
|
@ -370,3 +370,14 @@ async def test_a_direct_sdk_session_with_no_recorded_identity_still_warms_unattr
|
|||
await tick(llm_router, active=refresher(keys=FakeKeyDirectory({})))
|
||||
assert replayed_models(llm_router) == ["smart-claude"]
|
||||
assert llm_router.completion_calls[0]["metadata"]["user_api_key_team_id"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_replay_never_falls_back_to_a_group_warming_did_not_validate():
|
||||
"""Eligibility, every-member cacheability, affinity and pricing are properties of the target GROUP, so a
|
||||
fallback would substitute a group carrying none of them and spend the customer's money warming nothing."""
|
||||
llm_router, redis = warming_rig(redis=FakeRedisCache())
|
||||
seed_session(redis)
|
||||
await tick(llm_router)
|
||||
assert llm_router.completion_calls, "expected a replay"
|
||||
assert all(call["disable_fallbacks"] is True for call in llm_router.completion_calls)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue