diff --git a/litellm/main.py b/litellm/main.py index 42d5ecc8f20..8626d64ebd7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -27,6 +27,7 @@ from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args +from litellm._internal_context import is_resent_failure from litellm._logging import redact_string from litellm._uuid import uuid @@ -6242,6 +6243,10 @@ def completion( ) +def _is_not_resent_failure(error: BaseException) -> bool: + return not (isinstance(error, Exception) and is_resent_failure(error)) + + def completion_with_retries(*args, **kwargs): """ Executes a litellm.completion() with 3 retries @@ -6261,10 +6266,15 @@ def completion_with_retries(*args, **kwargs): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.Retrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return retryer(original_function, *args, **kwargs) @@ -6287,10 +6297,15 @@ async def acompletion_with_retries(*args, **kwargs): retryer = tenacity.AsyncRetrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.AsyncRetrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return await retryer(original_function, *args, **kwargs) @@ -6315,10 +6330,15 @@ def responses_with_retries(*args, **kwargs): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.Retrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return retryer(original_function, *args, **kwargs) @@ -6342,10 +6362,15 @@ async def aresponses_with_retries(*args, **kwargs): retryer = tenacity.AsyncRetrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.AsyncRetrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return await retryer(original_function, *args, **kwargs) diff --git a/litellm/utils.py b/litellm/utils.py index 4e1c6a91130..c2acb89b6e6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1980,7 +1980,8 @@ def client(original_function): retry_policy=kwargs.get("retry_policy"), ) kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops - litellm.num_retries = None # set retries to None to prevent infinite loops + if not is_resent_failure(e): + litellm.num_retries = None # set retries to None to prevent infinite loops context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) @@ -2015,7 +2016,8 @@ def client(original_function): retry_policy=kwargs.get("retry_policy"), ) kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops - litellm.num_retries = None # set retries to None to prevent infinite loops + if not is_resent_failure(e): + litellm.num_retries = None # set retries to None to prevent infinite loops is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( @@ -2314,8 +2316,9 @@ def client(original_function): elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" result = await litellm.acompletion_with_retries(*args, **kwargs) - except Exception: - pass + except Exception as retry_error: + if is_resent_failure(retry_error): + raise else: return result elif ( @@ -2348,8 +2351,9 @@ def client(original_function): elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" result = await litellm.aresponses_with_retries(*args, **kwargs) - except Exception: - pass + except Exception as retry_error: + if is_resent_failure(retry_error): + raise else: return result diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 1683b4a41e7..f02e7762deb 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -5608,9 +5608,10 @@ async def test_async_messages_with_num_retries_send_the_rejected_reasoning_only_ @pytest.mark.usefixtures("_chat_bridge_httpx") -def test_sync_messages_with_num_retries_send_the_rejected_reasoning_only_once( - respx_mock: respx.MockRouter, +def test_sync_messages_with_global_num_retries_send_the_rejected_reasoning_once_and_keep_the_setting( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "num_retries", 2) route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] ) @@ -5623,10 +5624,62 @@ def test_sync_messages_with_num_retries_send_the_rejected_reasoning_only_once( max_tokens=64, tools=_RETRY_TOOLS, messages=_history_with_foreign_reasoning(), - num_retries=2, ) assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 0] + assert litellm.num_retries == 2 + + +_SERVER_ERROR: Final = httpx.Response( + 500, json={"error": {"message": "The server had an error while processing your request.", "type": "server_error"}} +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_async_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_once_rejected( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_SERVER_ERROR, _RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 1, 0] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +def test_sync_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_once_rejected( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_SERVER_ERROR, _RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 1, 0] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] class _FailureLog(CustomLogger): diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 19de6ccd634..526ead24d26 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -747,6 +747,12 @@ def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]: def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool): + return _upstream_after(respx_mock, monkeypatch, (first,), stream) + + +def _upstream_after( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, failures: tuple[httpx.Response, ...], stream: bool +) -> respx.Route: monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") litellm.in_memory_llm_clients_cache.flush_cache() @@ -755,7 +761,7 @@ def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, fir if stream else httpx.Response(200, json=TEXT_RESPONSE) ) - return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[first, served]) + return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[*failures, served]) @pytest.mark.asyncio @@ -817,9 +823,10 @@ async def test_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_on assert _replayed_reasoning_counts(upstream) == [1, 0] -def test_sync_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_once( +def test_sync_openai_turn_with_global_num_retries_sends_the_rejected_reasoning_once_and_keeps_the_setting( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(litellm, "num_retries", 2) upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) litellm.anthropic.messages.create( @@ -827,10 +834,60 @@ def test_sync_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_onc messages=_history(replayed_reasoning=True), model="openai/gpt-5.6-luna", tools=[dict(BASH_TOOL)], - num_retries=2, ) assert _replayed_reasoning_counts(upstream) == [1, 0] + assert litellm.num_retries == 2 + + +_SERVER_ERROR: Final = { + "error": {"message": "The server had an error while processing your request.", "type": "server_error"} +} + + +@pytest.mark.asyncio +async def test_openai_turn_retries_a_server_error_with_the_reasoning_and_drops_it_only_once_rejected( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream_after( + respx_mock, + monkeypatch, + (httpx.Response(500, json=_SERVER_ERROR), httpx.Response(400, json=REJECTED_REPLAYED_REASONING)), + False, + ) + + response: Final = await litellm.anthropic.messages.acreate( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] + + +def test_sync_openai_turn_retries_a_server_error_with_the_reasoning_and_drops_it_only_once_rejected( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream_after( + respx_mock, + monkeypatch, + (httpx.Response(500, json=_SERVER_ERROR), httpx.Response(400, json=REJECTED_REPLAYED_REASONING)), + False, + ) + + response: Final = litellm.anthropic.messages.create( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] class _FailureLog(CustomLogger):