This commit is contained in:
Houyun Sun 2026-11-04 13:07:44 +00:00 • committed by GitHub
commit d17712a06c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 167 additions and 9 deletions

View file

@ -7623,9 +7623,30 @@ class PythonCore(DeprecatedRouterMembers):
)
return creating or healthy_deployments
@staticmethod
def _run_coroutine_in_background(coro: Awaitable[object]) -> None:
"""
Schedule a coroutine from sync code. Called from a thread with no running loop (the usual
sync Router call), so run it to completion there; inside a loop (sync code called from an
async handler) just queue it and let the loop do the work.
"""
try:
loop: Final = asyncio.get_running_loop()
except RuntimeError:
try:
asyncio.run(coro)
except Exception:
verbose_router_logger.debug("background coroutine failed", exc_info=True)
return
loop.create_task(coro)
def _generic_api_call_with_fallbacks(self, model: str, original_function: Callable, **kwargs):
"""
Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
This is the sync twin of _ageneric_api_call_with_fallbacks: async_function_with_fallbacks is
driven through run_async_function, and the deployment is picked inside
_generic_api_call_with_fallbacks_helper so every retry / fallback hop re-routes.
Args:
model: The model to use
original_function: The handler function to call (e.g., litellm.completion)
@ -7633,15 +7654,36 @@ class PythonCore(DeprecatedRouterMembers):
Returns:
The response from the handler function
"""
handler_name: Final = original_function.__name__
try:
kwargs["model"] = model
kwargs["original_generic_function"] = original_function
kwargs["original_function"] = self._generic_api_call_with_fallbacks_helper
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata")
verbose_router_logger.debug("Inside generic_api_call_with_fallbacks() - model: %s", model)
response: Final = self.function_with_fallbacks(**kwargs)
return response
except Exception as e:
# async_function_with_fallbacks can't be awaited from here, so fire the slack/teams
# alert the async twin sends. It no-ops unless router.slack_alerting_logger is set.
self._run_coroutine_in_background(
send_llm_exception_alert(
ctx=self._core_ctx,
request_kwargs=kwargs,
error_traceback_str=traceback.format_exc(),
original_exception=e,
)
)
raise e
def _generic_api_call_with_fallbacks_helper(self, model: str, original_generic_function: Callable, **kwargs):
"""
One attempt of the sync generic fallback chain (see _generic_api_call_with_fallbacks), routed
against the group the hop is for.
"""
handler_name: Final = getattr(original_generic_function, "__name__", "")
metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call")
try:
verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model)
self._update_kwargs_before_fallbacks(
model=model,
kwargs=kwargs,
metadata_variable_name=metadata_variable_name,
)
with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name):
deployment: Final = self.get_available_deployment(
model=model,
@ -7673,7 +7715,7 @@ class PythonCore(DeprecatedRouterMembers):
custom_llm_provider: Final = provider_for_generic_call(data)
response: Final = original_function(
response: Final = original_generic_function(
**{
**data,
"custom_llm_provider": custom_llm_provider,
@ -8608,10 +8650,19 @@ class PythonCore(DeprecatedRouterMembers):
):
def sync_wrapper(
model: str | None = None,
custom_llm_provider: str | None = None,
client: object | None = None,
**kwargs,
):
# model sits in the signature, so without this it never reaches kwargs and the
# retry/fallback loop has no group to route against. Don't forward
# custom_llm_provider when it's unset: the deployment already carries the provider
# and a None would shadow it.
if model is not None:
kwargs["model"] = model
if custom_llm_provider and "custom_llm_provider" not in kwargs:
kwargs["custom_llm_provider"] = custom_llm_provider
return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs)
return sync_wrapper

View file

@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
import httpx
import litellm
import pytest
import respx
from litellm import Router
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
@ -1259,3 +1260,108 @@ def test_router_model_group_encrypted_content_affinity_callback_registration():
assert encrypted_content_callbacks[0].ctx is router._core_ctx
finally:
router.discard()
@pytest.mark.respx(base_url="https://api.openai.com/v1")
def test_sync_responses_retries_on_the_router_num_retries(
respx_mock: respx.MockRouter,
) -> None:
"""Sync router.responses should retry a 503 like router.aresponses does (#49938)."""
route = respx_mock.post("/responses").mock(
side_effect=[
httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}),
httpx.Response(
200,
json={
"id": "resp_1",
"object": "response",
"created_at": 0,
"status": "completed",
"model": "gpt-4o",
"output": [],
"usage": {
"input_tokens": 1,
"output_tokens": 1,
"total_tokens": 2,
},
},
),
]
)
router = Router(
model_list=[
{
"model_name": "my-model",
"litellm_params": {
"model": "openai/gpt-4o",
"api_key": "sk-test",
},
}
],
num_retries=1,
retry_after=0,
disable_cooldowns=True,
)
try:
response = router.responses(model="my-model", input="hi")
finally:
router.retire()
assert route.call_count == 2
assert response.status == "completed"
@pytest.mark.respx(base_url="https://api.openai.com/v1")
def test_sync_responses_falls_back_to_the_next_group(
respx_mock: respx.MockRouter,
) -> None:
"""A 503 on the primary group should land on the backup group, asynchronously or not."""
primary = respx_mock.post("/primary/responses").mock(
return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}})
)
backup = respx_mock.post("/backup/responses").mock(
return_value=httpx.Response(
200,
json={
"id": "resp_2",
"object": "response",
"created_at": 0,
"status": "completed",
"model": "gpt-4o",
"output": [],
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
},
)
)
router = Router(
model_list=[
{
"model_name": "primary",
"litellm_params": {
"model": "openai/gpt-4o",
"api_key": "sk-test",
"api_base": "https://api.openai.com/v1/primary",
},
},
{
"model_name": "backup",
"litellm_params": {
"model": "openai/gpt-4o",
"api_key": "sk-test",
"api_base": "https://api.openai.com/v1/backup",
},
},
],
num_retries=0,
retry_after=0,
disable_cooldowns=True,
fallbacks=[{"primary": ["backup"]}],
)
try:
response = router.responses(model="primary", input="hi")
finally:
router.retire()
assert primary.call_count == 1
assert backup.call_count == 1
assert response.status == "completed"

View file

@ -3243,7 +3243,8 @@ def test_sync_generic_api_call_preserves_requested_model_group_in_logs():
original_function=mock_original_function,
)
assert response == {"status": "ok"}
# the response now goes through the fallback loop, which stamps attempt headers onto it
assert response["status"] == "ok"
assert captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6"
assert captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6"
assert captured_kwargs["litellm_metadata"]["deployment"] == "bedrock/global.anthropic.claude-sonnet-4-6"
@ -3288,7 +3289,7 @@ def test_sync_generic_api_call_uses_request_kwargs_for_deployment_selection():
allowed_model_region="eu",
)
assert response == {"status": "ok"}
assert response["status"] == "ok"
assert captured_kwargs["model"] == "anthropic/eu-model"
finally:
router.discard()