mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge b52e3d9e08 into f5e740e69a
This commit is contained in:
commit
d17712a06c
3 changed files with 167 additions and 9 deletions
|
|
@ -7623,9 +7623,30 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
return creating or healthy_deployments
|
||||
|
||||
@staticmethod
|
||||
def _run_coroutine_in_background(coro: Awaitable[object]) -> None:
|
||||
"""
|
||||
Schedule a coroutine from sync code. Called from a thread with no running loop (the usual
|
||||
sync Router call), so run it to completion there; inside a loop (sync code called from an
|
||||
async handler) just queue it and let the loop do the work.
|
||||
"""
|
||||
try:
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
try:
|
||||
asyncio.run(coro)
|
||||
except Exception:
|
||||
verbose_router_logger.debug("background coroutine failed", exc_info=True)
|
||||
return
|
||||
loop.create_task(coro)
|
||||
|
||||
def _generic_api_call_with_fallbacks(self, model: str, original_function: Callable, **kwargs):
|
||||
"""
|
||||
Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
|
||||
|
||||
This is the sync twin of _ageneric_api_call_with_fallbacks: async_function_with_fallbacks is
|
||||
driven through run_async_function, and the deployment is picked inside
|
||||
_generic_api_call_with_fallbacks_helper so every retry / fallback hop re-routes.
|
||||
Args:
|
||||
model: The model to use
|
||||
original_function: The handler function to call (e.g., litellm.completion)
|
||||
|
|
@ -7633,15 +7654,36 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
Returns:
|
||||
The response from the handler function
|
||||
"""
|
||||
handler_name: Final = original_function.__name__
|
||||
try:
|
||||
kwargs["model"] = model
|
||||
kwargs["original_generic_function"] = original_function
|
||||
kwargs["original_function"] = self._generic_api_call_with_fallbacks_helper
|
||||
self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs, metadata_variable_name="litellm_metadata")
|
||||
verbose_router_logger.debug("Inside generic_api_call_with_fallbacks() - model: %s", model)
|
||||
response: Final = self.function_with_fallbacks(**kwargs)
|
||||
return response
|
||||
except Exception as e:
|
||||
# async_function_with_fallbacks can't be awaited from here, so fire the slack/teams
|
||||
# alert the async twin sends. It no-ops unless router.slack_alerting_logger is set.
|
||||
self._run_coroutine_in_background(
|
||||
send_llm_exception_alert(
|
||||
ctx=self._core_ctx,
|
||||
request_kwargs=kwargs,
|
||||
error_traceback_str=traceback.format_exc(),
|
||||
original_exception=e,
|
||||
)
|
||||
)
|
||||
raise e
|
||||
|
||||
def _generic_api_call_with_fallbacks_helper(self, model: str, original_generic_function: Callable, **kwargs):
|
||||
"""
|
||||
One attempt of the sync generic fallback chain (see _generic_api_call_with_fallbacks), routed
|
||||
against the group the hop is for.
|
||||
"""
|
||||
handler_name: Final = getattr(original_generic_function, "__name__", "")
|
||||
metadata_variable_name: Final = get_router_metadata_variable_name(function_name="generic_api_call")
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _generic_api_call() - handler: %s, model: %s", handler_name, model)
|
||||
self._update_kwargs_before_fallbacks(
|
||||
model=model,
|
||||
kwargs=kwargs,
|
||||
metadata_variable_name=metadata_variable_name,
|
||||
)
|
||||
with self._video_call_pin(handler_name, kwargs), self._route_capability_pin(handler_name):
|
||||
deployment: Final = self.get_available_deployment(
|
||||
model=model,
|
||||
|
|
@ -7673,7 +7715,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
|
||||
custom_llm_provider: Final = provider_for_generic_call(data)
|
||||
|
||||
response: Final = original_function(
|
||||
response: Final = original_generic_function(
|
||||
**{
|
||||
**data,
|
||||
"custom_llm_provider": custom_llm_provider,
|
||||
|
|
@ -8608,10 +8650,19 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
):
|
||||
|
||||
def sync_wrapper(
|
||||
model: str | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
client: object | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
# model sits in the signature, so without this it never reaches kwargs and the
|
||||
# retry/fallback loop has no group to route against. Don't forward
|
||||
# custom_llm_provider when it's unset: the deployment already carries the provider
|
||||
# and a None would shadow it.
|
||||
if model is not None:
|
||||
kwargs["model"] = model
|
||||
if custom_llm_provider and "custom_llm_provider" not in kwargs:
|
||||
kwargs["custom_llm_provider"] = custom_llm_provider
|
||||
return self._generic_api_call_with_fallbacks(original_function=original_function, **kwargs)
|
||||
|
||||
return sync_wrapper
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
|||
import httpx
|
||||
import litellm
|
||||
import pytest
|
||||
import respx
|
||||
from litellm import Router
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
|
|
@ -1259,3 +1260,108 @@ def test_router_model_group_encrypted_content_affinity_callback_registration():
|
|||
assert encrypted_content_callbacks[0].ctx is router._core_ctx
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
@pytest.mark.respx(base_url="https://api.openai.com/v1")
|
||||
def test_sync_responses_retries_on_the_router_num_retries(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
"""Sync router.responses should retry a 503 like router.aresponses does (#49938)."""
|
||||
route = respx_mock.post("/responses").mock(
|
||||
side_effect=[
|
||||
httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}),
|
||||
httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"status": "completed",
|
||||
"model": "gpt-4o",
|
||||
"output": [],
|
||||
"usage": {
|
||||
"input_tokens": 1,
|
||||
"output_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "my-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "sk-test",
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=1,
|
||||
retry_after=0,
|
||||
disable_cooldowns=True,
|
||||
)
|
||||
try:
|
||||
response = router.responses(model="my-model", input="hi")
|
||||
finally:
|
||||
router.retire()
|
||||
|
||||
assert route.call_count == 2
|
||||
assert response.status == "completed"
|
||||
|
||||
|
||||
@pytest.mark.respx(base_url="https://api.openai.com/v1")
|
||||
def test_sync_responses_falls_back_to_the_next_group(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
"""A 503 on the primary group should land on the backup group, asynchronously or not."""
|
||||
primary = respx_mock.post("/primary/responses").mock(
|
||||
return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}})
|
||||
)
|
||||
backup = respx_mock.post("/backup/responses").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_2",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"status": "completed",
|
||||
"model": "gpt-4o",
|
||||
"output": [],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "primary",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "sk-test",
|
||||
"api_base": "https://api.openai.com/v1/primary",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "backup",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "sk-test",
|
||||
"api_base": "https://api.openai.com/v1/backup",
|
||||
},
|
||||
},
|
||||
],
|
||||
num_retries=0,
|
||||
retry_after=0,
|
||||
disable_cooldowns=True,
|
||||
fallbacks=[{"primary": ["backup"]}],
|
||||
)
|
||||
try:
|
||||
response = router.responses(model="primary", input="hi")
|
||||
finally:
|
||||
router.retire()
|
||||
|
||||
assert primary.call_count == 1
|
||||
assert backup.call_count == 1
|
||||
assert response.status == "completed"
|
||||
|
|
|
|||
|
|
@ -3243,7 +3243,8 @@ def test_sync_generic_api_call_preserves_requested_model_group_in_logs():
|
|||
original_function=mock_original_function,
|
||||
)
|
||||
|
||||
assert response == {"status": "ok"}
|
||||
# the response now goes through the fallback loop, which stamps attempt headers onto it
|
||||
assert response["status"] == "ok"
|
||||
assert captured_kwargs["model"] == "bedrock/global.anthropic.claude-sonnet-4-6"
|
||||
assert captured_kwargs["litellm_metadata"]["model_group"] == "claude-sonnet-4-6"
|
||||
assert captured_kwargs["litellm_metadata"]["deployment"] == "bedrock/global.anthropic.claude-sonnet-4-6"
|
||||
|
|
@ -3288,7 +3289,7 @@ def test_sync_generic_api_call_uses_request_kwargs_for_deployment_selection():
|
|||
allowed_model_region="eu",
|
||||
)
|
||||
|
||||
assert response == {"status": "ok"}
|
||||
assert response["status"] == "ok"
|
||||
assert captured_kwargs["model"] == "anthropic/eu-model"
|
||||
finally:
|
||||
router.discard()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue