scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-25 05:54:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 125

Merge 12d87a9888 into b690225956

Browse source
This commit is contained in:
thibault-linktree 2026-10-20 04:22:22 +00:00 • committed by GitHub
commit dfb974d09a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 279 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
litellm/router.py
View file

@ -565,6 +565,7 @@ _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str])
_RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None)
_ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_DEPLOYMENTS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_DEPLOYMENT_PARAMS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | GenericLiteLLMParams]] = TypeAdapter(
Mapping[str, object] | GenericLiteLLMParams
@ -8654,6 +8655,10 @@ class Router:
num_retries = _retry_policy_retries
_retry_policy_applies = True
first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry(
exception=original_exception,
already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
)
# raises an exception if this error should not be retries
# Skip this check if retry policy applies (retry policy takes precedence)
if not _retry_policy_applies:
@ -8664,6 +8669,7 @@ class Router:
context_window_fallbacks=context_window_fallbacks,
regular_fallbacks=fallbacks,
content_policy_fallbacks=content_policy_fallbacks,
tried_deployment_ids=first_skipped_ids,
)
# Update max_retries after overrides (deployment_num_retries / retry_policy)
_metadata["max_retries"] = num_retries
@ -8671,10 +8677,6 @@ class Router:
## LOGGING
if num_retries > 0:
kwargs = self.log_retry(kwargs=kwargs, e=original_exception)
first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry(
exception=original_exception,
already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
)
if first_skipped_ids:
kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids # rebind-ok: the next attempt reads it
else:
@ -8710,6 +8712,8 @@ class Router:
return response
except Exception as e:
if is_guardrail_intervention(e):
raise
# Always track the latest error so we raise the most
# recent exception instead of the first one.
original_exception = e
@ -8733,6 +8737,10 @@ class Router:
else:
_healthy_deployments = []
skipped_ids = self._deployment_ids_to_skip_on_retry(
exception=e,
already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
)
# Check if this error is non-retryable (e.g., 400 context
# window exceeded). If so, raise immediately instead of
# continuing the retry loop. Respect retry policy
@ -8746,14 +8754,11 @@ class Router:
context_window_fallbacks=context_window_fallbacks,
regular_fallbacks=fallbacks,
content_policy_fallbacks=content_policy_fallbacks,
tried_deployment_ids=skipped_ids,
)
except Exception:
raise e
skipped_ids = self._deployment_ids_to_skip_on_retry(
exception=e,
already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
)
if skipped_ids:
kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it
_timeout = self._time_to_sleep_before_retry(
@ -8825,10 +8830,13 @@ class Router:
context_window_fallbacks: list | None = None,
content_policy_fallbacks: list | None = None,
regular_fallbacks: list | None = None,
tried_deployment_ids: tuple[str, ...] = (),
):
"""
1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None
3. raise a non-retryable error (e.g. 400, 404) unless a healthy deployment that has not yet
refused this request (`tried_deployment_ids`) can take the retry
2. raise an exception for RateLimitError if
- there are no fallbacks
@ -8850,13 +8858,15 @@ class Router:
raise error
status_code: Final = getattr(error, "status_code", None)
if status_code is not None and not litellm.should_retry(status_code):
# 401/403 are special cases - allow retry if multiple deployments exist (handled below)
if status_code not in (401, 403):
# 401/403 are special cases - allow retry if multiple deployments exist (handled below)
if status_code is not None and not litellm.should_retry(status_code) and status_code not in (401, 403):
deployments: Final = _DEPLOYMENTS_ADAPTER.validate_python(healthy_deployments or ())
has_untried_deployment: Final = bool(tried_deployment_ids) and any(
_configured_model_info(deployment).get("id") not in tried_deployment_ids for deployment in deployments
)
if not has_untried_deployment:
raise error
if isinstance(error, litellm.NotFoundError):
raise error
return True
# Error we should only retry if there are other deployments
if isinstance(error, openai.RateLimitError):
if (

151
tests/unit/test_router/test_router.py
View file

@ -20376,6 +20376,153 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info=
}
_UPSTREAM_404 = {
"message": "The model `gpt-5.6` does not exist or you do not have access to it.",
"type": "invalid_request_error",
"code": "model_not_found",
}
_UPSTREAM_SUCCESS = {
"id": "chatcmpl-failover",
"object": "chat.completion",
"created": 1,
"model": "gpt-5.6",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi back"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("status_code,upstream_error", [(404, _UPSTREAM_404), (400, _UPSTREAM_400)])
async def test_router_non_retryable_error_fails_over_to_sibling_deployment(
monkeypatch: pytest.MonkeyPatch, status_code, upstream_error
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(
model_list=[
_retry_skip_deployment("rejecting", "rejecting", litellm_params={"weight": 1}),
_retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}),
],
num_retries=2,
)
with respx.mock(assert_all_called=True) as respx_mock:
rejecting = respx_mock.post("https://rejecting.local/v1/chat/completions").mock(
return_value=httpx.Response(status_code, json={"error": upstream_error})
)
accepting = respx_mock.post("https://accepting.local/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS)
)
response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert (rejecting.call_count, accepting.call_count) == (1, 1)
assert response.choices[0].message.content == "hi back"
assert response._hidden_params["model_id"] == "accepting"
assert response._hidden_params["additional_headers"]["x-litellm-attempted-retries"] == 1
@pytest.mark.asyncio
async def test_router_not_found_failover_still_cools_down_the_refusing_deployment(monkeypatch: pytest.MonkeyPatch):
from litellm.router_utils.cooldown_handlers import async_get_cooldown_deployments
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(
model_list=[
_retry_skip_deployment("rejecting", "rejecting", litellm_params={"weight": 1}),
_retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}),
],
num_retries=2,
)
with respx.mock(assert_all_called=True) as respx_mock:
respx_mock.post("https://rejecting.local/v1/chat/completions").mock(
return_value=httpx.Response(404, json={"error": _UPSTREAM_404})
)
respx_mock.post("https://accepting.local/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS)
)
await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == [
"rejecting"
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"status_code,upstream_error,expected_exception",
[(404, _UPSTREAM_404, litellm.NotFoundError), (400, _UPSTREAM_400, litellm.BadRequestError)],
)
async def test_router_non_retryable_error_on_single_deployment_group_is_not_retried(
monkeypatch: pytest.MonkeyPatch, status_code, upstream_error, expected_exception
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(model_list=[_retry_skip_deployment("only", "only")], num_retries=2)
with respx.mock(assert_all_called=True) as respx_mock:
only = respx_mock.post("https://only.local/v1/chat/completions").mock(
return_value=httpx.Response(status_code, json={"error": upstream_error})
)
with pytest.raises(expected_exception):
await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert only.call_count == 1
@pytest.mark.asyncio
async def test_router_non_retryable_error_fails_over_past_every_refusing_deployment(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(
model_list=[
_retry_skip_deployment("first", "first", litellm_params={"weight": 1}),
_retry_skip_deployment("second", "second", litellm_params={"weight": 1}),
_retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}),
],
num_retries=2,
)
with respx.mock(assert_all_called=True) as respx_mock:
refusing = tuple(
respx_mock.post(f"https://{host}.local/v1/chat/completions").mock(
return_value=httpx.Response(404, json={"error": _UPSTREAM_404})
)
for host in ("first", "second")
)
accepting = respx_mock.post("https://accepting.local/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS)
)
response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert tuple(route.call_count for route in (*refusing, accepting)) == (1, 1, 1)
assert response._hidden_params["model_id"] == "accepting"
@pytest.mark.asyncio
async def test_router_non_retryable_error_stops_once_every_deployment_refused(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(
model_list=[
_retry_skip_deployment("first", "first", litellm_params={"weight": 1}),
_retry_skip_deployment("second", "second", litellm_params={"weight": 0}),
],
num_retries=3,
disable_cooldowns=True,
)
with respx.mock(assert_all_called=True) as respx_mock:
first = respx_mock.post("https://first.local/v1/chat/completions").mock(
return_value=httpx.Response(400, json={"error": _UPSTREAM_400})
)
second = respx_mock.post("https://second.local/v1/chat/completions").mock(
return_value=httpx.Response(400, json={"error": _UPSTREAM_400})
)
with pytest.raises(litellm.BadRequestError) as exc_info:
await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert (first.call_count, second.call_count) == (1, 1)
assert exc_info.value.failed_deployment_id == "second"
@pytest.mark.parametrize(
"status_code,failed_deployment_id,already_skipped,expected",
[
@ -21167,7 +21314,7 @@ async def test_router_max_parallel_requests_overflow_is_429_without_cooldown_or_
assert all("capped-deployment" in r.message and "max_parallel_requests=1" in r.message for r in rejected)
assert route.call_count == 1
assert sibling_route.call_count == 0
assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == []
assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == []
@pytest.mark.asyncio
@ -21286,7 +21433,7 @@ async def test_router_max_parallel_requests_overflow_takes_the_ordinary_429_fall
assert len(results) == 3
assert primary.call_count == 1
assert fallback.call_count == 2
assert await _async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == []
assert await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) == []
@pytest.mark.asyncio

108
tests/unit/test_router/test_router_retries.py
View file

@ -12,6 +12,7 @@ from pydantic import TypeAdapter
import litellm
from litellm import Router
from litellm.exceptions import GuardrailRaisedException
from litellm.integrations.custom_logger import CustomLogger
from litellm.router import AllowedFailsPolicy, RetryPolicy
import os
@ -629,7 +630,7 @@ def test_no_retry_for_bad_request_error_400():
Test that 400 BadRequestError is NOT retried, even if healthy deployments exist.
This tests the fix for GitHub issue #19216.
"""
healthy_deployments = ["deployment1", "deployment2"] # Multiple healthy deployments
healthy_deployments = _deployments("deployment1", "deployment2")
router = Router(
model_list=[
@ -666,7 +667,7 @@ def test_no_retry_for_unprocessable_entity_error_422():
"""
Test that 422 UnprocessableEntityError is NOT retried, even if healthy deployments exist.
"""
healthy_deployments = ["deployment1", "deployment2"] # Multiple healthy deployments
healthy_deployments = _deployments("deployment1", "deployment2")
router = Router(
model_list=[
@ -703,6 +704,109 @@ def test_no_retry_for_unprocessable_entity_error_422():
print("Correctly raised UnprocessableEntityError without retry:", e)
def _not_found() -> litellm.NotFoundError:
return litellm.NotFoundError(message="model not found", model="m", llm_provider="openai")
def _bad_request() -> litellm.BadRequestError:
return litellm.BadRequestError(message="bad request", model="m", llm_provider="openai")
def _deployments(*deployment_ids: str) -> list[dict[str, dict[str, str]]]:
return [{"model_info": {"id": deployment_id}} for deployment_id in deployment_ids]
@pytest.mark.parametrize(
"error,tried_deployment_ids,healthy_deployment_ids",
[
(_not_found(), ("a",), ("a", "b")),
(_bad_request(), ("a",), ("a", "b")),
(_bad_request(), ("a",), ("b",)),
(
litellm.UnprocessableEntityError(
message="unprocessable",
model="m",
llm_provider="openai",
response=httpx.Response(422, request=httpx.Request("POST", "https://example.invalid/v1")),
),
("a",),
("a", "b"),
),
],
)
def test_non_retryable_error_is_retried_on_an_untried_healthy_deployment(
error: Exception, tried_deployment_ids: tuple[str, ...], healthy_deployment_ids: tuple[str, ...]
) -> None:
healthy_deployments: Final = _deployments(*healthy_deployment_ids)
assert (
_retry_test_router().should_retry_this_error(
error=error,
healthy_deployments=healthy_deployments,
all_deployments=healthy_deployments,
tried_deployment_ids=tried_deployment_ids,
)
is True
)
@pytest.mark.parametrize(
"error,tried_deployment_ids,healthy_deployment_ids",
[
(_bad_request(), (), ("a", "b")),
(_bad_request(), ("a", "b"), ("a", "b")),
(_not_found(), ("a",), ("a",)),
(_not_found(), ("a",), ()),
],
)
def test_non_retryable_error_is_raised_when_no_untried_healthy_deployment_is_left(
error: Exception, tried_deployment_ids: tuple[str, ...], healthy_deployment_ids: tuple[str, ...]
) -> None:
healthy_deployments: Final = _deployments(*healthy_deployment_ids)
with pytest.raises(type(error)) as exc_info:
_retry_test_router().should_retry_this_error(
error=error,
healthy_deployments=healthy_deployments,
all_deployments=healthy_deployments,
tried_deployment_ids=tried_deployment_ids,
)
assert exc_info.value is error
@pytest.mark.asyncio
async def test_guardrail_rejection_on_a_retry_is_not_failed_over_to_a_sibling() -> None:
router: Final = Router(
model_list=[
{
"model_name": "retry-test-model",
"litellm_params": {"model": "openai/retry-test-model", "api_key": "test-key"},
"model_info": {"id": deployment_id},
}
for deployment_id in ("refusing", "guarded", "unguarded")
],
disable_cooldowns=True,
)
not_found: Final = _not_found()
not_found.retry_skip_deployment_id = "refusing"
guardrail_rejection: Final = GuardrailRaisedException(guardrail_name="pii", message="blocked", blocked_content=True)
guardrail_rejection.retry_skip_deployment_id = "guarded"
attempts: Final = AsyncMock(side_effect=[not_found, guardrail_rejection, "unguarded answer"])
with pytest.raises(GuardrailRaisedException) as exc_info:
await router.async_function_with_retries(
model="retry-test-model",
messages=[{"role": "user", "content": "retry test"}],
original_function=attempts,
num_retries=2,
metadata={},
)
assert exc_info.value is guardrail_rejection
assert attempts.await_count == 2
def test_no_retry_when_no_healthy_deployments():
healthy_deployments = []