This commit is contained in:
thibault-linktree 2026-11-02 10:26:42 +11:00 • committed by GitHub
commit 3729833fc9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 293 additions and 23 deletions

View file

@ -9578,6 +9578,10 @@ class PythonCore(DeprecatedRouterMembers):
num_retries = _retry_policy_retries
_retry_policy_applies = True
first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry(
exception=original_exception,
already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
)
# raises an exception if this error should not be retries
# Skip this check if retry policy applies (retry policy takes precedence)
if not _retry_policy_applies:
@ -9588,6 +9592,7 @@ class PythonCore(DeprecatedRouterMembers):
context_window_fallbacks=context_window_fallbacks,
regular_fallbacks=fallbacks,
content_policy_fallbacks=content_policy_fallbacks,
tried_deployment_ids=first_skipped_ids,
)
# Update max_retries after overrides (deployment_num_retries / retry_policy)
_metadata["max_retries"] = num_retries
@ -9595,10 +9600,6 @@ class PythonCore(DeprecatedRouterMembers):
## LOGGING
if num_retries > 0:
kwargs = self._core_log_retry(kwargs=kwargs, e=original_exception)
first_skipped_ids: Final = self._deployment_ids_to_skip_on_retry(
exception=original_exception,
already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
)
if first_skipped_ids:
kwargs["_retry_skipped_deployment_ids"] = first_skipped_ids # rebind-ok: the next attempt reads it
if isinstance(
@ -9640,6 +9641,8 @@ class PythonCore(DeprecatedRouterMembers):
return response
except Exception as e:
if is_guardrail_intervention(e):
raise
# Always track the latest error so we raise the most
# recent exception instead of the first one.
original_exception = e
@ -9663,6 +9666,10 @@ class PythonCore(DeprecatedRouterMembers):
else:
_healthy_deployments = []
skipped_ids = self._deployment_ids_to_skip_on_retry(
exception=e,
already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
)
# Check if this error is non-retryable (e.g., 400 context
# window exceeded). If so, raise immediately instead of
# continuing the retry loop. Respect retry policy
@ -9676,14 +9683,11 @@ class PythonCore(DeprecatedRouterMembers):
context_window_fallbacks=context_window_fallbacks,
regular_fallbacks=fallbacks,
content_policy_fallbacks=content_policy_fallbacks,
tried_deployment_ids=skipped_ids,
)
except Exception:
raise e
skipped_ids = self._deployment_ids_to_skip_on_retry(
exception=e,
already_skipped=kwargs.get("_retry_skipped_deployment_ids"),
)
if skipped_ids:
kwargs["_retry_skipped_deployment_ids"] = skipped_ids # rebind-ok: the next attempt reads it
if isinstance(e, litellm_exceptions.UnsupportedEndpointError) and _every_deployment_refused(
@ -9777,10 +9781,13 @@ class PythonCore(DeprecatedRouterMembers):
context_window_fallbacks: list | None = None,
content_policy_fallbacks: list | None = None,
regular_fallbacks: list | None = None,
tried_deployment_ids: tuple[str, ...] = (),
):
"""
1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None
3. raise a non-retryable error (e.g. 400, 404) unless a healthy deployment that has not yet
refused this request (`tried_deployment_ids`) can take the retry
2. raise an exception for RateLimitError if
- there are no fallbacks
@ -9802,13 +9809,18 @@ class PythonCore(DeprecatedRouterMembers):
raise error
status_code: Final = getattr(error, "status_code", None)
if status_code is not None and not self._host.invoke("should_retry")(status_code):
# 401/403 are special cases - allow retry if multiple deployments exist (handled below)
if status_code not in (401, 403) and not isinstance(error, litellm_exceptions.UnsupportedEndpointError):
# 401/403 are special cases - allow retry if multiple deployments exist (handled below)
if (
status_code is not None
and not self._host.invoke("should_retry")(status_code)
and status_code not in (401, 403)
):
has_untried_deployment: Final = bool(tried_deployment_ids) and not _every_deployment_refused(
_DEPLOYMENTS_ADAPTER.validate_python(healthy_deployments or ()), tried_deployment_ids
)
if not has_untried_deployment:
raise error
if isinstance(error, litellm_exceptions.NotFoundError):
raise error
return True
# Error we should only retry if there are other deployments
if isinstance(error, openai.RateLimitError):
if (

View file

@ -179,13 +179,10 @@ async def test_lone_deployment_retries_only_retryable_errors(
assert len(host.calls_to("acompletion")) == expected_calls
@pytest.mark.parametrize(
("builder", "expected_calls"),
[(bad_request, 1), (context_window, 1), (content_policy, 1), (server_error, 3), (timeout, 3)],
)
@pytest.mark.parametrize("builder", [server_error, timeout])
@pytest.mark.asyncio
async def test_two_deployment_group_retries_only_retryable_errors(
make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder, expected_calls: int
async def test_two_deployment_group_retries_retryable_errors_up_to_the_budget(
make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder
) -> None:
core: Final = make_core(model_list=two_deployments(), num_retries=2)
host.answer("*", fresh(builder))
@ -193,7 +190,21 @@ async def test_two_deployment_group_retries_only_retryable_errors(
error: Final = await attempt(core)
assert type(error) is type(builder())
assert len(host.calls_to("acompletion")) == expected_calls
assert len(host.calls_to("acompletion")) == 3
@pytest.mark.parametrize("builder", [bad_request, context_window, content_policy])
@pytest.mark.asyncio
async def test_non_retryable_error_tries_each_deployment_of_the_group_once(
make_core: MakeCore, host: ScriptedHost, builder: ErrorBuilder
) -> None:
core: Final = make_core(model_list=two_deployments(), num_retries=2)
host.answer("*", fresh(builder))
error: Final = await attempt(core)
assert type(error) is type(builder())
assert sorted(host.deployments_called("acompletion")) == ["a", "b"]
@pytest.mark.asyncio

View file

@ -21954,6 +21954,149 @@ def _retry_skip_deployment(deployment_id, host, litellm_params=None, model_info=
}
_UPSTREAM_404 = {
"message": "The model `gpt-5.6` does not exist or you do not have access to it.",
"type": "invalid_request_error",
"code": "model_not_found",
}
_UPSTREAM_SUCCESS = {
"id": "chatcmpl-failover",
"object": "chat.completion",
"created": 1,
"model": "gpt-5.6",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi back"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
@pytest.mark.asyncio
@pytest.mark.parametrize("status_code,upstream_error", [(404, _UPSTREAM_404), (400, _UPSTREAM_400)])
async def test_router_non_retryable_error_fails_over_to_sibling_deployment(
monkeypatch: pytest.MonkeyPatch, status_code, upstream_error
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(
model_list=[
_retry_skip_deployment("rejecting", "rejecting", litellm_params={"weight": 1}),
_retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}),
],
num_retries=2,
)
with respx.mock(assert_all_called=True) as respx_mock:
rejecting = respx_mock.post("https://rejecting.local/v1/chat/completions").mock(
return_value=httpx.Response(status_code, json={"error": upstream_error})
)
accepting = respx_mock.post("https://accepting.local/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS)
)
response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert (rejecting.call_count, accepting.call_count) == (1, 1)
assert response.choices[0].message.content == "hi back"
assert response._hidden_params["model_id"] == "accepting"
assert response._hidden_params["additional_headers"]["x-litellm-attempted-retries"] == 1
@pytest.mark.asyncio
async def test_router_not_found_failover_still_cools_down_the_refusing_deployment(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(
model_list=[
_retry_skip_deployment("rejecting", "rejecting", litellm_params={"weight": 1}),
_retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}),
],
num_retries=2,
)
with respx.mock(assert_all_called=True) as respx_mock:
respx_mock.post("https://rejecting.local/v1/chat/completions").mock(
return_value=httpx.Response(404, json={"error": _UPSTREAM_404})
)
respx_mock.post("https://accepting.local/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS)
)
await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert await async_get_cooldown_deployments(ctx=router._core_ctx, parent_otel_span=None) == ["rejecting"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"status_code,upstream_error,expected_exception",
[(404, _UPSTREAM_404, litellm.NotFoundError), (400, _UPSTREAM_400, litellm.BadRequestError)],
)
async def test_router_non_retryable_error_on_single_deployment_group_is_not_retried(
monkeypatch: pytest.MonkeyPatch, status_code, upstream_error, expected_exception
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(model_list=[_retry_skip_deployment("only", "only")], num_retries=2)
with respx.mock(assert_all_called=True) as respx_mock:
only = respx_mock.post("https://only.local/v1/chat/completions").mock(
return_value=httpx.Response(status_code, json={"error": upstream_error})
)
with pytest.raises(expected_exception):
await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert only.call_count == 1
@pytest.mark.asyncio
async def test_router_non_retryable_error_fails_over_past_every_refusing_deployment(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(
model_list=[
_retry_skip_deployment("first", "first", litellm_params={"weight": 1}),
_retry_skip_deployment("second", "second", litellm_params={"weight": 1}),
_retry_skip_deployment("accepting", "accepting", litellm_params={"weight": 0}),
],
num_retries=2,
)
with respx.mock(assert_all_called=True) as respx_mock:
refusing = tuple(
respx_mock.post(f"https://{host}.local/v1/chat/completions").mock(
return_value=httpx.Response(404, json={"error": _UPSTREAM_404})
)
for host in ("first", "second")
)
accepting = respx_mock.post("https://accepting.local/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_UPSTREAM_SUCCESS)
)
response = await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert tuple(route.call_count for route in (*refusing, accepting)) == (1, 1, 1)
assert response._hidden_params["model_id"] == "accepting"
@pytest.mark.asyncio
async def test_router_non_retryable_error_stops_once_every_deployment_refused(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
router = litellm.Router(
model_list=[
_retry_skip_deployment("first", "first", litellm_params={"weight": 1}),
_retry_skip_deployment("second", "second", litellm_params={"weight": 0}),
],
num_retries=3,
disable_cooldowns=True,
)
with respx.mock(assert_all_called=True) as respx_mock:
first = respx_mock.post("https://first.local/v1/chat/completions").mock(
return_value=httpx.Response(400, json={"error": _UPSTREAM_400})
)
second = respx_mock.post("https://second.local/v1/chat/completions").mock(
return_value=httpx.Response(400, json={"error": _UPSTREAM_400})
)
with pytest.raises(litellm.BadRequestError) as exc_info:
await router.acompletion(model="gpt-5.6", messages=[{"role": "user", "content": "hi"}])
assert (first.call_count, second.call_count) == (1, 1)
assert exc_info.value.failed_deployment_id == "second"
@pytest.mark.parametrize(
"status_code,failed_deployment_id,already_skipped,expected",
[

View file

@ -12,6 +12,7 @@ from pydantic import TypeAdapter
import litellm
from litellm import Router
from litellm.exceptions import GuardrailRaisedException
from litellm.integrations.custom_logger import CustomLogger
from litellm.router import AllowedFailsPolicy, RetryPolicy
import os
@ -629,7 +630,7 @@ def test_no_retry_for_bad_request_error_400():
Test that 400 BadRequestError is NOT retried, even if healthy deployments exist.
This tests the fix for GitHub issue #19216.
"""
healthy_deployments = ["deployment1", "deployment2"] # Multiple healthy deployments
healthy_deployments = _deployments("deployment1", "deployment2")
router = Router(
model_list=[
@ -666,7 +667,7 @@ def test_no_retry_for_unprocessable_entity_error_422():
"""
Test that 422 UnprocessableEntityError is NOT retried, even if healthy deployments exist.
"""
healthy_deployments = ["deployment1", "deployment2"] # Multiple healthy deployments
healthy_deployments = _deployments("deployment1", "deployment2")
router = Router(
model_list=[
@ -703,6 +704,109 @@ def test_no_retry_for_unprocessable_entity_error_422():
print("Correctly raised UnprocessableEntityError without retry:", e)
def _not_found() -> litellm.NotFoundError:
return litellm.NotFoundError(message="model not found", model="m", llm_provider="openai")
def _bad_request() -> litellm.BadRequestError:
return litellm.BadRequestError(message="bad request", model="m", llm_provider="openai")
def _deployments(*deployment_ids: str) -> list[dict[str, dict[str, str]]]:
return [{"model_info": {"id": deployment_id}} for deployment_id in deployment_ids]
@pytest.mark.parametrize(
"error,tried_deployment_ids,healthy_deployment_ids",
[
(_not_found(), ("a",), ("a", "b")),
(_bad_request(), ("a",), ("a", "b")),
(_bad_request(), ("a",), ("b",)),
(
litellm.UnprocessableEntityError(
message="unprocessable",
model="m",
llm_provider="openai",
response=httpx.Response(422, request=httpx.Request("POST", "https://example.invalid/v1")),
),
("a",),
("a", "b"),
),
],
)
def test_non_retryable_error_is_retried_on_an_untried_healthy_deployment(
error: Exception, tried_deployment_ids: tuple[str, ...], healthy_deployment_ids: tuple[str, ...]
) -> None:
healthy_deployments: Final = _deployments(*healthy_deployment_ids)
assert (
_retry_test_router().should_retry_this_error(
error=error,
healthy_deployments=healthy_deployments,
all_deployments=healthy_deployments,
tried_deployment_ids=tried_deployment_ids,
)
is True
)
@pytest.mark.parametrize(
"error,tried_deployment_ids,healthy_deployment_ids",
[
(_bad_request(), (), ("a", "b")),
(_bad_request(), ("a", "b"), ("a", "b")),
(_not_found(), ("a",), ("a",)),
(_not_found(), ("a",), ()),
],
)
def test_non_retryable_error_is_raised_when_no_untried_healthy_deployment_is_left(
error: Exception, tried_deployment_ids: tuple[str, ...], healthy_deployment_ids: tuple[str, ...]
) -> None:
healthy_deployments: Final = _deployments(*healthy_deployment_ids)
with pytest.raises(type(error)) as exc_info:
_retry_test_router().should_retry_this_error(
error=error,
healthy_deployments=healthy_deployments,
all_deployments=healthy_deployments,
tried_deployment_ids=tried_deployment_ids,
)
assert exc_info.value is error
@pytest.mark.asyncio
async def test_guardrail_rejection_on_a_retry_is_not_failed_over_to_a_sibling() -> None:
router: Final = Router(
model_list=[
{
"model_name": "retry-test-model",
"litellm_params": {"model": "openai/retry-test-model", "api_key": "test-key"},
"model_info": {"id": deployment_id},
}
for deployment_id in ("refusing", "guarded", "unguarded")
],
disable_cooldowns=True,
)
not_found: Final = _not_found()
not_found.retry_skip_deployment_id = "refusing"
guardrail_rejection: Final = GuardrailRaisedException(guardrail_name="pii", message="blocked", blocked_content=True)
guardrail_rejection.retry_skip_deployment_id = "guarded"
attempts: Final = AsyncMock(side_effect=[not_found, guardrail_rejection, "unguarded answer"])
with pytest.raises(GuardrailRaisedException) as exc_info:
await router.async_function_with_retries(
model="retry-test-model",
messages=[{"role": "user", "content": "retry test"}],
original_function=attempts,
num_retries=2,
metadata={},
)
assert exc_info.value is guardrail_rejection
assert attempts.await_count == 2
def test_no_retry_when_no_healthy_deployments():
healthy_deployments = []