diff --git a/litellm/router.py b/litellm/router.py index bc0b1280d12..45fadcbab4a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -558,6 +558,11 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( ) +def _without_target_order(kwargs: Mapping[str, object]) -> Mapping[str, object]: + """Drop the router-internal order-fallback target so it never reaches a provider call.""" + return MappingProxyType({k: v for k, v in kwargs.items() if k != "_target_order"}) + + class Router: model_names: set = set() cache_responses: bool | None = False @@ -2172,9 +2177,8 @@ class Router: "messages": messages, "caching": self.cache_responses, "client": model_client, - **kwargs, + **_without_target_order(kwargs), } - input_kwargs.pop("_target_order", None) response: Final = litellm.completion(**input_kwargs) verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -3194,9 +3198,8 @@ class Router: "messages": messages, "caching": self.cache_responses, "client": model_client, - **kwargs, + **_without_target_order(kwargs), } - input_kwargs.pop("_target_order", None) input_kwargs.pop("silent_model", None) input_kwargs.pop("include_fallback_errors", None) @@ -4072,7 +4075,7 @@ class Router: "prompt": prompt, "caching": self.cache_responses, "client": model_client, - **kwargs, + **_without_target_order(kwargs), } ) self.success_calls[model_name] += 1 @@ -4132,7 +4135,7 @@ class Router: "prompt": prompt, "caching": self.cache_responses, "client": model_client, - **kwargs, + **_without_target_order(kwargs), } ) @@ -4236,7 +4239,7 @@ class Router: "file": file, "caching": self.cache_responses, "client": model_client, - **kwargs, + **_without_target_order(kwargs), } ) @@ -4889,7 +4892,7 @@ class Router: response_kwargs: Final = { **data, "caching": self.cache_responses, - **kwargs, + **_without_target_order(kwargs), "model": model_name, } # Only set custom_llm_provider if it's not None @@ -5339,7 +5342,7 @@ class Router: **data, "custom_llm_provider": custom_llm_provider, "caching": self.cache_responses, - **kwargs, + **_without_target_order(kwargs), } ) @@ -5405,7 +5408,7 @@ class Router: "input": input, "caching": self.cache_responses, "client": model_client, - **kwargs, + **_without_target_order(kwargs), } ) self.success_calls[model_name] += 1 @@ -5468,7 +5471,7 @@ class Router: "input": input, "caching": self.cache_responses, "client": model_client, - **kwargs, + **_without_target_order(kwargs), } ) diff --git a/tests/test_litellm/test_router_order_fallback.py b/tests/test_litellm/test_router_order_fallback.py index d74e0a6ffa4..042d724df0b 100644 --- a/tests/test_litellm/test_router_order_fallback.py +++ b/tests/test_litellm/test_router_order_fallback.py @@ -552,35 +552,48 @@ async def test_router_order_fallback_retries_keep_target_order(): assert seen_target_orders.count(2) >= 2 +@pytest.mark.asyncio +async def test_generic_api_call_strips_target_order_from_provider_kwargs(): + captured: Final = {} + + async def _fake_provider(**provider_kwargs): + captured.update(provider_kwargs) + return "ok" + + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": {"model": "gpt-4o", "api_key": "key", "order": 2}, + "model_info": {"id": "2"}, + }, + ], + ) + response = await router._ageneric_api_call_with_fallbacks_helper( + model="test-model", + original_generic_function=_fake_provider, + _target_order=2, + messages=[{"role": "user", "content": "hi"}], + ) + assert response == "ok" + assert captured["model"] == "gpt-4o" + assert "_target_order" not in captured + + def test_check_non_standard_fallback_format(): from litellm.router_utils.fallback_event_handlers import ( _check_non_standard_fallback_format, ) # Standard formats - assert ( - _check_non_standard_fallback_format([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) - == False - ) + assert _check_non_standard_fallback_format([{"gpt-3.5-turbo": ["claude-3-haiku"]}]) == False assert _check_non_standard_fallback_format([{"model": ["qwen-backup"]}]) == False - assert ( - _check_non_standard_fallback_format( - [{"model": ["qwen-backup"], "region": ["us-east-1"]}] - ) - == False - ) + assert _check_non_standard_fallback_format([{"model": ["qwen-backup"], "region": ["us-east-1"]}]) == False # Non-standard formats assert _check_non_standard_fallback_format([{"model": "qwen-backup"}]) == True assert ( - _check_non_standard_fallback_format( - [{"model": "qwen-backup", "messages": [{"role": "user", "content": "hi"}]}] - ) - == True - ) - assert ( - _check_non_standard_fallback_format( - [{"model": ["qwen-backup"], "api_key": "some-key"}] - ) + _check_non_standard_fallback_format([{"model": "qwen-backup", "messages": [{"role": "user", "content": "hi"}]}]) == True ) + assert _check_non_standard_fallback_format([{"model": ["qwen-backup"], "api_key": "some-key"}]) == True