diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index b9a90c7ea2a..d1386f612c0 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -7614,9 +7614,30 @@ class PythonCore(DeprecatedRouterMembers): ) return creating or healthy_deployments + @staticmethod + def _run_coroutine_in_background(coro: Awaitable[object]) -> None: + """ + Schedule a coroutine from sync code. Called from a thread with no running loop (the usual + sync Router call), so run it to completion there; inside a loop (sync code called from an + async handler) just queue it and let the loop do the work. + """ + try: + loop: Final = asyncio.get_running_loop() + except RuntimeError: + try: + asyncio.run(coro) + except Exception: + verbose_router_logger.debug("background coroutine failed", exc_info=True) + return + loop.create_task(coro) + def _generic_api_call_with_fallbacks(self, model: str, original_function: Callable, **kwargs): """ Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router + + This is the sync twin of _ageneric_api_call_with_fallbacks: async_function_with_fallbacks is + driven through run_async_function, and the deployment is picked inside + _generic_api_call_with_fallbacks_helper so every retry / fallback hop re-routes. Args: model: The model to use original_function: The handler function to call (e.g., litellm.completion) @@ -7624,15 +7645,36 @@ class PythonCore(DeprecatedRouterMembers): Returns: The response from the handler function """ - handler_name: Final = original_function.__name__ + try: + kwargs["model"] = model + kwargs["original_generic_function"] = original_function + kwargs["original_function"] = self._generic_api_call_with_fallbacks_helper + self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata") + verbose_router_logger.debug("Inside generic_api_call_with_fallbacks() - model: %s", model) + response: Final = self.function_with_fallbacks(**kwargs) + return response + except Exception as e: + # async_function_with_fallbacks can't be awaited from here, so fire the slack/teams + # alert the async twin sends. It no-ops unless router.slack_alerting_logger is set. + self._run_coroutine_in_background( + send_llm_exception_alert( + ctx=self._core_ctx, + request_kwargs=kwargs, + error_traceback_str=traceback.format_exc(), + original_exception=e, + ) + ) + raise e + + def _generic_api_call_with_fallbacks_helper(self, model: str, original_generic_function: Callable, **kwargs): + """ + One attempt of the sync generic fallback chain (see _generic_api_call_with_fallbacks), routed + against the group the hop is for. + """ + handler_name: Final = getattr(original_generic_function, "__name__", "") metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call") try: verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model) - self._update_kwargs_before_fallbacks( - model=model, - kwargs=kwargs, - metadata_variable_name=metadata_variable_name, - ) with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name): deployment: Final = self.get_available_deployment( model=model, @@ -7664,7 +7706,7 @@ class PythonCore(DeprecatedRouterMembers): custom_llm_provider: Final = provider_for_generic_call(data) - response: Final = original_function( + response: Final = original_generic_function( **{ **data, "custom_llm_provider": custom_llm_provider, @@ -8599,10 +8641,19 @@ class PythonCore(DeprecatedRouterMembers): ): def sync_wrapper( + model: str | None = None, custom_llm_provider: str | None = None, client: object | None = None, **kwargs, ): + # model sits in the signature, so without this it never reaches kwargs and the + # retry/fallback loop has no group to route against. Don't forward + # custom_llm_provider when it's unset: the deployment already carries the provider + # and a None would shadow it. + if model is not None: + kwargs["model"] = model + if custom_llm_provider and "custom_llm_provider" not in kwargs: + kwargs["custom_llm_provider"] = custom_llm_provider return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs) return sync_wrapper diff --git a/tests/unit/test_router/test_router_endpoints.py b/tests/unit/test_router/test_router_endpoints.py index 0c57fb662ac..0175d4ac940 100644 --- a/tests/unit/test_router/test_router_endpoints.py +++ b/tests/unit/test_router/test_router_endpoints.py @@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import litellm import pytest +import respx from litellm import Router from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -1259,3 +1260,108 @@ def test_router_model_group_encrypted_content_affinity_callback_registration(): assert encrypted_content_callbacks[0].ctx is router._core_ctx finally: router.discard() + + +@pytest.mark.respx(base_url="https://api.openai.com/v1") +def test_sync_responses_retries_on_the_router_num_retries( + respx_mock: respx.MockRouter, +) -> None: + """Sync router.responses should retry a 503 like router.aresponses does (#49938).""" + route = respx_mock.post("/responses").mock( + side_effect=[ + httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}), + httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4o", + "output": [], + "usage": { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + }, + }, + ), + ] + ) + router = Router( + model_list=[ + { + "model_name": "my-model", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-test", + }, + } + ], + num_retries=1, + retry_after=0, + disable_cooldowns=True, + ) + try: + response = router.responses(model="my-model", input="hi") + finally: + router.retire() + + assert route.call_count == 2 + assert response.status == "completed" + + +@pytest.mark.respx(base_url="https://api.openai.com/v1") +def test_sync_responses_falls_back_to_the_next_group( + respx_mock: respx.MockRouter, +) -> None: + """A 503 on the primary group should land on the backup group, asynchronously or not.""" + primary = respx_mock.post("/primary/responses").mock( + return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}) + ) + backup = respx_mock.post("/backup/responses").mock( + return_value=httpx.Response( + 200, + json={ + "id": "resp_2", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-4o", + "output": [], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + ) + router = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-test", + "api_base": "https://api.openai.com/v1/primary", + }, + }, + { + "model_name": "backup", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-test", + "api_base": "https://api.openai.com/v1/backup", + }, + }, + ], + num_retries=0, + retry_after=0, + disable_cooldowns=True, + fallbacks=[{"primary": ["backup"]}], + ) + try: + response = router.responses(model="primary", input="hi") + finally: + router.retire() + + assert primary.call_count == 1 + assert backup.call_count == 1 + assert response.status == "completed" diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 5ad6eb819ef..a91d58de313 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -3243,7 +3243,8 @@ def test_sync_generic_api_call_preserves_requested_model_group_in_logs(): original_function=mock_original_function, ) - assert response == {"status": "ok"} + # the response now goes through the fallback loop, which stamps attempt headers onto it + assert response["status"] == "ok" assert captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6" assert captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6" assert captured_kwargs["litellm_metadata"]["deployment"] == "bedrock/global.anthropic.claude-sonnet-4-6" @@ -3288,7 +3289,7 @@ def test_sync_generic_api_call_uses_request_kwargs_for_deployment_selection(): allowed_model_region="eu", ) - assert response == {"status": "ok"} + assert response["status"] == "ok" assert captured_kwargs["model"] == "anthropic/eu-model" finally: router.discard()