diff --git a/litellm/router.py b/litellm/router.py index 45fadcbab4a..c93c1753f0e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -558,11 +558,6 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( ) -def _without_target_order(kwargs: Mapping[str, object]) -> Mapping[str, object]: - """Drop the router-internal order-fallback target so it never reaches a provider call.""" - return MappingProxyType({k: v for k, v in kwargs.items() if k != "_target_order"}) - - class Router: model_names: set = set() cache_responses: bool | None = False @@ -2177,7 +2172,7 @@ class Router: "messages": messages, "caching": self.cache_responses, "client": model_client, - **_without_target_order(kwargs), + **kwargs, } response: Final = litellm.completion(**input_kwargs) verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -3198,7 +3193,7 @@ class Router: "messages": messages, "caching": self.cache_responses, "client": model_client, - **_without_target_order(kwargs), + **kwargs, } input_kwargs.pop("silent_model", None) input_kwargs.pop("include_fallback_errors", None) @@ -4075,7 +4070,7 @@ class Router: "prompt": prompt, "caching": self.cache_responses, "client": model_client, - **_without_target_order(kwargs), + **kwargs, } ) self.success_calls[model_name] += 1 @@ -4135,7 +4130,7 @@ class Router: "prompt": prompt, "caching": self.cache_responses, "client": model_client, - **_without_target_order(kwargs), + **kwargs, } ) @@ -4239,7 +4234,7 @@ class Router: "file": file, "caching": self.cache_responses, "client": model_client, - **_without_target_order(kwargs), + **kwargs, } ) @@ -4892,7 +4887,7 @@ class Router: response_kwargs: Final = { **data, "caching": self.cache_responses, - **_without_target_order(kwargs), + **kwargs, "model": model_name, } # Only set custom_llm_provider if it's not None @@ -5342,7 +5337,7 @@ class Router: **data, "custom_llm_provider": custom_llm_provider, "caching": self.cache_responses, - **_without_target_order(kwargs), + **kwargs, } ) @@ -5408,7 +5403,7 @@ class Router: "input": input, "caching": self.cache_responses, "client": model_client, - **_without_target_order(kwargs), + **kwargs, } ) self.success_calls[model_name] += 1 @@ -5471,7 +5466,7 @@ class Router: "input": input, "caching": self.cache_responses, "client": model_client, - **_without_target_order(kwargs), + **kwargs, } ) @@ -11933,7 +11928,7 @@ class Router: ) ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) - _target_order: Final = (request_kwargs or {}).get("_target_order") + _target_order: Final = (request_kwargs or {}).pop("_target_order", None) healthy_deployments = litellm.utils._get_order_filtered_deployments( cast(list[dict], healthy_deployments), target_order=_target_order ) @@ -12698,7 +12693,7 @@ class Router: ) ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) - _target_order: Final = (request_kwargs or {}).get("_target_order") + _target_order: Final = (request_kwargs or {}).pop("_target_order", None) healthy_deployments = litellm.utils._get_order_filtered_deployments( healthy_deployments, target_order=_target_order ) diff --git a/tests/test_litellm/test_router_order_fallback.py b/tests/test_litellm/test_router_order_fallback.py index 042d724df0b..8b9075f845c 100644 --- a/tests/test_litellm/test_router_order_fallback.py +++ b/tests/test_litellm/test_router_order_fallback.py @@ -6,8 +6,10 @@ should be tried first, and higher order deployments should be used as fallbacks when lower order deployments fail. """ +import json from typing import Final, Optional +import httpx import pytest import litellm @@ -580,6 +582,64 @@ async def test_generic_api_call_strips_target_order_from_provider_kwargs(): assert "_target_order" not in captured +@pytest.mark.asyncio +async def test_text_completion_order_fallback_hop_does_not_send_target_order_upstream(): + upstream_bodies: Final[list[dict]] = [] + + def _upstream(request: httpx.Request) -> httpx.Response: + upstream_bodies.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "cmpl-1", + "object": "text_completion", + "created": 0, + "model": "gpt-3.5-turbo-instruct", + "choices": [{"text": "ok from order 2", "index": 0, "logprobs": None, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + session: Final = httpx.AsyncClient(transport=httpx.MockTransport(_upstream)) + litellm.in_memory_llm_clients_cache.flush_cache() + litellm.aclient_session = session + router = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "text-completion-openai/gpt-3.5-turbo-instruct", + "api_key": "key", + "mock_response": Exception("fail order 1"), + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "text-completion-openai/gpt-3.5-turbo-instruct", + "api_key": "key", + "api_base": "http://upstream.test", + "order": 2, + }, + "model_info": {"id": "2"}, + }, + ], + num_retries=0, + ) + try: + response = await router.atext_completion(model="test-model", prompt="hi") + finally: + litellm.aclient_session = None + litellm.in_memory_llm_clients_cache.flush_cache() + await session.aclose() + + assert response._hidden_params["model_id"] == "2" + assert upstream_bodies + assert all("_target_order" not in body for body in upstream_bodies) + + def test_check_non_standard_fallback_format(): from litellm.router_utils.fallback_event_handlers import ( _check_non_standard_fallback_format,