scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 79

fix(router): strip _target_order at every provider boundary via a shared helper

Browse source
This commit is contained in:
ryan-crabbe-berri 2026-09-01 14:34:48 -07:00
parent c7212e7fe2
commit ac964918c5
2 changed files with 46 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
litellm/router.py
View file

@ -558,6 +558,11 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset(
)
def _without_target_order(kwargs: Mapping[str, object]) -> Mapping[str, object]:
"""Drop the router-internal order-fallback target so it never reaches a provider call."""
return MappingProxyType({k: v for k, v in kwargs.items() if k != "_target_order"})
class Router:
model_names: set = set()
cache_responses: bool | None = False
@ -2172,9 +2177,8 @@ class Router:
"messages": messages,
"caching": self.cache_responses,
"client": model_client,
**kwargs,
**_without_target_order(kwargs),
}
input_kwargs.pop("_target_order", None)
response: Final = litellm.completion(**input_kwargs)
verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
@ -3194,9 +3198,8 @@ class Router:
"messages": messages,
"caching": self.cache_responses,
"client": model_client,
**kwargs,
**_without_target_order(kwargs),
}
input_kwargs.pop("_target_order", None)
input_kwargs.pop("silent_model", None)
input_kwargs.pop("include_fallback_errors", None)
@ -4072,7 +4075,7 @@ class Router:
"prompt": prompt,
"caching": self.cache_responses,
"client": model_client,
**kwargs,
**_without_target_order(kwargs),
}
)
self.success_calls[model_name] += 1
@ -4132,7 +4135,7 @@ class Router:
"prompt": prompt,
"caching": self.cache_responses,
"client": model_client,
**kwargs,
**_without_target_order(kwargs),
}
)
@ -4236,7 +4239,7 @@ class Router:
"file": file,
"caching": self.cache_responses,
"client": model_client,
**kwargs,
**_without_target_order(kwargs),
}
)
@ -4889,7 +4892,7 @@ class Router:
response_kwargs: Final = {
**data,
"caching": self.cache_responses,
**kwargs,
**_without_target_order(kwargs),
"model": model_name,
}
# Only set custom_llm_provider if it's not None
@ -5339,7 +5342,7 @@ class Router:
**data,
"custom_llm_provider": custom_llm_provider,
"caching": self.cache_responses,
**kwargs,
**_without_target_order(kwargs),
}
)
@ -5405,7 +5408,7 @@ class Router:
"input": input,
"caching": self.cache_responses,
"client": model_client,
**kwargs,
**_without_target_order(kwargs),
}
)
self.success_calls[model_name] += 1
@ -5468,7 +5471,7 @@ class Router:
"input": input,
"caching": self.cache_responses,
"client": model_client,
**kwargs,
**_without_target_order(kwargs),
}
)

51
tests/test_litellm/test_router_order_fallback.py
View file

@ -552,35 +552,48 @@ async def test_router_order_fallback_retries_keep_target_order():
assert seen_target_orders.count(2) >= 2
@pytest.mark.asyncio
async def test_generic_api_call_strips_target_order_from_provider_kwargs():
captured: Final = {}
async def _fake_provider(**provider_kwargs):
captured.update(provider_kwargs)
return "ok"
router = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 2},
"model_info": {"id": "2"},
},
],
)
response = await router._ageneric_api_call_with_fallbacks_helper(
model="test-model",
original_generic_function=_fake_provider,
_target_order=2,
messages=[{"role": "user", "content": "hi"}],
)
assert response == "ok"
assert captured["model"] == "gpt-4o"
assert "_target_order" not in captured
def test_check_non_standard_fallback_format():
from litellm.router_utils.fallback_event_handlers import (
_check_non_standard_fallback_format,
)
# Standard formats
assert (
_check_non_standard_fallback_format([{"gpt-3.5-turbo": ["claude-3-haiku"]}])
== False
)
assert _check_non_standard_fallback_format([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) == False
assert _check_non_standard_fallback_format([{"model": ["qwen-backup"]}]) == False
assert (
_check_non_standard_fallback_format(
[{"model": ["qwen-backup"], "region": ["us-east-1"]}]
)
== False
)
assert _check_non_standard_fallback_format([{"model": ["qwen-backup"], "region": ["us-east-1"]}]) == False
# Non-standard formats
assert _check_non_standard_fallback_format([{"model": "qwen-backup"}]) == True
assert (
_check_non_standard_fallback_format(
[{"model": "qwen-backup", "messages": [{"role": "user", "content": "hi"}]}]
)
== True
)
assert (
_check_non_standard_fallback_format(
[{"model": ["qwen-backup"], "api_key": "some-key"}]
)
_check_non_standard_fallback_format([{"model": "qwen-backup", "messages": [{"role": "user", "content": "hi"}]}])
== True
)
assert _check_non_standard_fallback_format([{"model": ["qwen-backup"], "api_key": "some-key"}]) == True