From b603e9293f1a85384b2a74886857b7436366411d Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Fri, 30 Oct 2026 11:54:40 -0700 Subject: [PATCH 1/5] fix(decisions): honor SDK num_retries --- litellm/utils.py | 22 ++++++++--- tests/unit/test_utils.py | 82 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 2ca8a49f968..aacd4c249c8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2218,7 +2218,7 @@ def client(original_function): # RETURN RESULT return result except Exception as e: - if call_type == CallTypes.completion.value: + if call_type in (CallTypes.completion.value, CallTypes.decisions.value, CallTypes.systemone.value): num_retries = kwargs.get("num_retries", None) or litellm.num_retries or None if kwargs.get("retry_policy", None): get_num_retries_from_retry_policy: Callable[..., int | None] = ( @@ -2243,9 +2243,16 @@ def client(original_function): or isinstance(e, openai.APIConnectionError) ): kwargs["num_retries"] = num_retries - return litellm.completion_with_retries(*args, **kwargs) + return litellm.completion_with_retries( + *args, + original_function=original_function + if call_type != CallTypes.completion.value + else litellm.completion, + **kwargs, + ) elif ( - isinstance(e, litellm.exceptions.ContextWindowExceededError) + call_type == CallTypes.completion.value + and isinstance(e, litellm.exceptions.ContextWindowExceededError) and context_window_fallback_dict and model in context_window_fallback_dict and not is_completion_litellm_router_call @@ -2548,7 +2555,11 @@ def client(original_function): retry_call_type: Final = _get_call_type_from_public_name(original_function.__name__) num_retries, kwargs = _get_wrapper_num_retries(kwargs=kwargs, exception=e) - if retry_call_type == CallTypes.acompletion.value: + if retry_call_type in ( + CallTypes.acompletion.value, + CallTypes.adecisions.value, + CallTypes.asystemone.value, + ): context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) @@ -2570,7 +2581,8 @@ def client(original_function): else: return result elif ( - isinstance(e, litellm.exceptions.ContextWindowExceededError) + retry_call_type == CallTypes.acompletion.value + and isinstance(e, litellm.exceptions.ContextWindowExceededError) and context_window_fallback_dict and model in context_window_fallback_dict and not is_acompletion_litellm_router_call diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0f471b1de71..b1a2c8ac2a0 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -5713,6 +5713,84 @@ def test_responses_uses_sdk_retries_without_router(monkeypatch: pytest.MonkeyPat litellm.in_memory_llm_clients_cache.flush_cache() +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("systemone_format", [False, True], ids=["decisions", "systemone"]) +@pytest.mark.parametrize( + "num_retries,upstream_failures,expected_attempts,router_call", + [ + pytest.param(2, 1, 2, False, id="recovers"), + pytest.param(2, 3, 3, False, id="exhausts"), + pytest.param(0, 1, 1, False, id="disabled"), + pytest.param(None, 1, 1, False, id="omitted"), + pytest.param(2, 1, 1, True, id="router-owns-retries"), + ], +) +async def test_decisions_sdk_retries( + monkeypatch: pytest.MonkeyPatch, + is_async: bool, + systemone_format: bool, + num_retries: int | None, + upstream_failures: int, + expected_attempts: int, + router_call: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + retry_kwargs: Final = {} if num_retries is None else {"num_retries": num_retries} + call: Final = ( + partial( + litellm.asystemone if is_async else litellm.systemone, + model="typesafe/jev-latest", + state="I was charged twice.", + questions={"needs_refund": {"type": "noul", "instructions": "Is a refund needed?"}}, + ) + if systemone_format + else partial( + litellm.adecisions if is_async else litellm.decisions, + model="typesafe/jev-latest", + input="I was charged twice.", + questions=[{"name": "needs_refund", "type": "predicate", "instructions": "Is a refund needed?"}], + ) + ) + request: Final = partial( + call, + api_base="https://decisions-retry.local", + api_key="test-key", + metadata={"model_group": "decisions-retry"} if router_call else None, + **retry_kwargs, + ) + responses: Final = tuple( + httpx.Response(503, json={"error": {"message": "temporary outage", "type": "server_error"}}) + for _ in range(upstream_failures) + ) + ( + httpx.Response( + 200, + json={"model": "jev-latest", "answers": {"needs_refund": {"type": "noul", "noul": 0.9}}}, + ), + ) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://decisions-retry.local/v1/systemone").mock(side_effect=responses) + if upstream_failures < expected_attempts: + pending: Final = request() + result: Final = await pending if asyncio.iscoroutine(pending) else pending + assert result.model_dump()["answers"] == ( + {"needs_refund": {"type": "noul", "noul": 0.9}} + if systemone_format + else ({"name": "needs_refund", "type": "predicate", "probability": 0.9},) + ) + elif is_async: + with pytest.raises(litellm.ServiceUnavailableError, match="temporary outage"): + await request() + else: + with pytest.raises(litellm.ServiceUnavailableError, match="temporary outage"): + request() + + assert route.call_count == expected_attempts + bodies: Final = tuple(json.loads(request.content) for request, _ in route.calls) + assert all(body == bodies[0] for body in bodies) + assert "num_retries" not in bodies[0] + + @pytest.mark.asyncio async def test_wrapper_async_replays_cached_converted_chat_stream_as_stream( monkeypatch: pytest.MonkeyPatch, @@ -11143,7 +11221,7 @@ async def test_responses_retry_on_auth_error(sync_mode, respx_mock: respx.MockRo model="gpt-4o", input="Test input", num_retries=num_retries, - api_key="sk-invalid-key-12345", + api_key="test-key", ) except Exception: pass @@ -11158,7 +11236,7 @@ async def test_responses_retry_on_auth_error(sync_mode, respx_mock: respx.MockRo model="gpt-4o", input="Test input", num_retries=num_retries, - api_key="sk-invalid-key-12345", + api_key="test-key", ) except Exception: pass From e21010703f529e6d6d1b29036b21b060b46ed1ae Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Fri, 30 Oct 2026 12:34:37 -0700 Subject: [PATCH 2/5] refactor(decisions): centralize SDK retry call pairs --- litellm/utils.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index aacd4c249c8..10aa5f6c887 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1689,6 +1689,15 @@ async def _client_async_logging_helper( ) +_COMPLETION_RETRY_CALL_PAIRS: Final[Mapping[CallTypes, CallTypes]] = MappingProxyType( + { + CallTypes.completion: CallTypes.acompletion, + CallTypes.decisions: CallTypes.adecisions, + CallTypes.systemone: CallTypes.asystemone, + } +) + + def _get_wrapper_num_retries(kwargs: dict[str, Any], exception: Exception) -> tuple[int | None, dict[str, Any]]: """ Get the number of retries from the kwargs and the retry policy. @@ -2218,7 +2227,7 @@ def client(original_function): # RETURN RESULT return result except Exception as e: - if call_type in (CallTypes.completion.value, CallTypes.decisions.value, CallTypes.systemone.value): + if call_type in _COMPLETION_RETRY_CALL_PAIRS: num_retries = kwargs.get("num_retries", None) or litellm.num_retries or None if kwargs.get("retry_policy", None): get_num_retries_from_retry_policy: Callable[..., int | None] = ( @@ -2555,11 +2564,7 @@ def client(original_function): retry_call_type: Final = _get_call_type_from_public_name(original_function.__name__) num_retries, kwargs = _get_wrapper_num_retries(kwargs=kwargs, exception=e) - if retry_call_type in ( - CallTypes.acompletion.value, - CallTypes.adecisions.value, - CallTypes.asystemone.value, - ): + if retry_call_type in _COMPLETION_RETRY_CALL_PAIRS.values(): context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) From cce78a4f6077183db6ba264aa7a8a13d41f7f93e Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Fri, 30 Oct 2026 14:24:38 -0700 Subject: [PATCH 3/5] fix(sdk): separate Decisions retries and retry async Messages calls --- litellm/utils.py | 98 +++++++++++++++++++++++------ tests/unit/test_utils.py | 129 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 207 insertions(+), 20 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 10aa5f6c887..a6d76802179 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -307,7 +307,7 @@ except (ImportError, AttributeError, TypeError): # Convert to str (if necessary) claude_json_str = json.dumps(json_data) import importlib.metadata -from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized +from collections.abc import AsyncIterator, Awaitable, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from typing_extensions import assert_never @@ -1689,9 +1689,8 @@ async def _client_async_logging_helper( ) -_COMPLETION_RETRY_CALL_PAIRS: Final[Mapping[CallTypes, CallTypes]] = MappingProxyType( +_DECISION_RETRY_CALL_PAIRS: Final[Mapping[CallTypes, CallTypes]] = MappingProxyType( { - CallTypes.completion: CallTypes.acompletion, CallTypes.decisions: CallTypes.adecisions, CallTypes.systemone: CallTypes.asystemone, } @@ -1998,6 +1997,26 @@ def _is_litellm_router_call(kwargs: Mapping[str, object], *, is_async: bool) -> return any(isinstance(bucket, Mapping) and "model_group" in bucket for bucket in metadata_buckets) +def _sdk_retry_kwargs( + kwargs: Mapping[str, object], num_retries: int | None, error: Exception, *, is_async: bool +) -> Mapping[str, object] | None: + if not num_retries or _is_litellm_router_call(kwargs, is_async=is_async): + return None + if isinstance(error, ImportError): + return None + if not is_async and not isinstance(error, openai.APIError): + return None + return MappingProxyType( + { + **kwargs, + "num_retries": num_retries, + "retry_strategy": "exponential_backoff_retry" + if is_async and isinstance(error, openai.RateLimitError) + else "constant_retry", + } + ) + + def client(original_function): from litellm.litellm_core_utils.core_helpers import max_retries_per_request_hit @@ -2227,7 +2246,10 @@ def client(original_function): # RETURN RESULT return result except Exception as e: - if call_type in _COMPLETION_RETRY_CALL_PAIRS: + completion_retry: Final = cast( # cast-ok: legacy retry helper forwards arbitrary SDK arguments + "Callable[..., object]", litellm.completion_with_retries + ) + if call_type == CallTypes.completion.value: num_retries = kwargs.get("num_retries", None) or litellm.num_retries or None if kwargs.get("retry_policy", None): get_num_retries_from_retry_policy: Callable[..., int | None] = ( @@ -2252,16 +2274,9 @@ def client(original_function): or isinstance(e, openai.APIConnectionError) ): kwargs["num_retries"] = num_retries - return litellm.completion_with_retries( - *args, - original_function=original_function - if call_type != CallTypes.completion.value - else litellm.completion, - **kwargs, - ) + return completion_retry(*args, **kwargs) elif ( - call_type == CallTypes.completion.value - and isinstance(e, litellm.exceptions.ContextWindowExceededError) + isinstance(e, litellm.exceptions.ContextWindowExceededError) and context_window_fallback_dict and model in context_window_fallback_dict and not is_completion_litellm_router_call @@ -2294,6 +2309,16 @@ def client(original_function): ): kwargs["num_retries"] = num_retries return litellm.responses_with_retries(*args, **kwargs) + elif call_type in _DECISION_RETRY_CALL_PAIRS: + decision_request_kwargs: Final[dict[str, object]] = {**kwargs} # mutable-ok: resets retry policy + decision_num_retries, decision_kwargs = _get_wrapper_num_retries( + kwargs=decision_request_kwargs, exception=e + ) + decision_retry_kwargs: Final = _sdk_retry_kwargs( + decision_kwargs, decision_num_retries, e, is_async=False + ) + if decision_retry_kwargs is not None: + return completion_retry(*args, original_function=original_function, **decision_retry_kwargs) traceback_exception: Final = traceback.format_exc() end_time = datetime.datetime.now() @@ -2431,11 +2456,21 @@ def client(original_function): print_verbose(f"Error while checking max token limit: {e}") # MODEL CALL - call_kwargs: Final = ( + model_call_kwargs: Final[dict[str, object]] = ( # mutable-ok: hooks require a dict {**kwargs, "input": _caching_handler_response.embedding_uncached_input} if _caching_handler_response is not None and _caching_handler_response.embedding_uncached_input is not None - else kwargs + else {**kwargs} + ) + call_kwargs: Final[dict[str, object]] = ( # mutable-ok: hooks require a dict + {**model_call_kwargs, "num_retries": 0, "max_retries": 0, "retry_policy": None} + if call_type == CallTypes.anthropic_messages.value + and ( + model_call_kwargs.get("num_retries") is not None + or litellm.num_retries is not None + or model_call_kwargs.get("retry_policy") is not None + ) + else model_call_kwargs ) try: result = await original_function(*args, **call_kwargs) @@ -2562,9 +2597,12 @@ def client(original_function): except Exception as e: raise e + async_completion_retry: Final = cast( # cast-ok: legacy retry helper forwards arbitrary SDK arguments + "Callable[..., Awaitable[object]]", litellm.acompletion_with_retries + ) retry_call_type: Final = _get_call_type_from_public_name(original_function.__name__) num_retries, kwargs = _get_wrapper_num_retries(kwargs=kwargs, exception=e) - if retry_call_type in _COMPLETION_RETRY_CALL_PAIRS.values(): + if retry_call_type == CallTypes.acompletion.value: context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) @@ -2580,14 +2618,13 @@ def client(original_function): kwargs["retry_strategy"] = "exponential_backoff_retry" elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" - result = await litellm.acompletion_with_retries(*args, **kwargs) + result = await async_completion_retry(*args, **kwargs) except Exception: pass else: return result elif ( - retry_call_type == CallTypes.acompletion.value - and isinstance(e, litellm.exceptions.ContextWindowExceededError) + isinstance(e, litellm.exceptions.ContextWindowExceededError) and context_window_fallback_dict and model in context_window_fallback_dict and not is_acompletion_litellm_router_call @@ -2618,6 +2655,29 @@ def client(original_function): else: return result + elif retry_call_type in _DECISION_RETRY_CALL_PAIRS.values(): + decision_retry_kwargs: Final = _sdk_retry_kwargs(kwargs, num_retries, e, is_async=True) + if decision_retry_kwargs is not None: + try: + result = await async_completion_retry( # rebind-ok: finally needs the retried stream for correlation cleanup + *args, original_function=original_function, **decision_retry_kwargs + ) + except Exception: # noqa: BLE001, S110 # preserve the original SDK error and annotate its retry metadata below + pass + else: + return result + elif retry_call_type == CallTypes.anthropic_messages.value: + messages_retry_kwargs: Final = _sdk_retry_kwargs(kwargs, num_retries, e, is_async=True) + if messages_retry_kwargs is not None: + try: + result = await async_completion_retry( # rebind-ok: finally needs the retried stream for correlation cleanup + *args, original_function=original_function, **messages_retry_kwargs + ) + except Exception: # noqa: BLE001, S110 # preserve the original SDK error and annotate its retry metadata below + pass + else: + return result + deployment_num_retries: Final = kwargs.get("num_retries") if deployment_num_retries is not None: setattr(e, "num_retries", deployment_num_retries) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b1a2c8ac2a0..8a902adb0bc 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -13,7 +13,7 @@ import queue import threading import time import uuid -from collections.abc import Callable, Iterator, Mapping +from collections.abc import AsyncIterable, Callable, Iterator, Mapping from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone from functools import partial @@ -5791,6 +5791,133 @@ async def test_decisions_sdk_retries( assert "num_retries" not in bodies[0] +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["complete", "stream"]) +@pytest.mark.parametrize( + "num_retries,global_retries,upstream_failures,expected_attempts,router_call", + [ + pytest.param(2, None, 1, 2, False, id="recovers"), + pytest.param(2, None, 3, 3, False, id="exhausts"), + pytest.param(0, None, 1, 1, False, id="disabled"), + pytest.param(None, None, 1, 1, False, id="omitted"), + pytest.param(2, None, 1, 1, True, id="router-owns-retries"), + pytest.param(None, 2, 1, 2, False, id="global-default"), + pytest.param(0, 2, 1, 1, False, id="disabled-overrides-global"), + ], +) +async def test_messages_sdk_retries( + monkeypatch: pytest.MonkeyPatch, + stream: bool, + num_retries: int | None, + global_retries: int | None, + upstream_failures: int, + expected_attempts: int, + router_call: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "num_retries", global_retries) + retry_kwargs: Final = {} if num_retries is None else {"num_retries": num_retries} + body: Final = { + "id": "msg_retry", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "retry succeeded"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 4, "output_tokens": 2}, + } + events: Final = ( + {"type": "message_start", "message": {**body, "content": []}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "retry succeeded"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 2}}, + {"type": "message_stop"}, + ) + success: Final = ( + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events), + ) + if stream + else httpx.Response(200, json=body) + ) + responses: Final = tuple( + httpx.Response( + 503, json={"type": "error", "error": {"type": "overloaded_error", "message": "temporary outage"}} + ) + for _ in range(upstream_failures) + ) + (success,) + request: Final = partial( + litellm.anthropic.messages.acreate, + model="anthropic/claude-sonnet-5-5", + max_tokens=16, + messages=[{"role": "user", "content": "hello"}], + api_base="https://messages-retry.local", + api_key="test-key", + max_retries=0, + stream=stream, + litellm_metadata={"model_group": "messages-retry"} if router_call else None, + **retry_kwargs, + ) + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://messages-retry.local/v1/messages").mock(side_effect=responses) + if upstream_failures < expected_attempts: + response: Final = await request() + if stream: + assert isinstance(response, AsyncIterable) + chunks: Final = [chunk async for chunk in response] + raw: Final = b"".join(chunk for chunk in chunks if isinstance(chunk, bytes)) + assert b"retry succeeded" in raw + assert raw.count(b"event: message_start") == 1 + assert b"event: message_stop" in raw + else: + assert isinstance(response, Mapping) + assert response["content"] == body["content"] + else: + with pytest.raises(litellm.InternalServerError, match="temporary outage"): + await request() + + assert route.call_count == expected_attempts + bodies: Final = tuple(json.loads(request.content) for request, _ in route.calls) + assert all(body == bodies[0] for body in bodies) + assert "num_retries" not in bodies[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model,endpoint", [("mistral/mistral-small-latest", "chat/completions"), ("openai/gpt-5.6", "responses")] +) +@pytest.mark.parametrize("num_retries,global_retries,expected_attempts", [(2, None, 3), (None, 2, 3), (0, 2, 1)]) +async def test_messages_bridge_has_one_sdk_retry_budget( + monkeypatch: pytest.MonkeyPatch, + model: str, + endpoint: str, + num_retries: int | None, + global_retries: int | None, + expected_attempts: int, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "num_retries", global_retries) + retry_kwargs: Final = {} if num_retries is None else {"num_retries": num_retries} + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post(f"https://messages-bridge-retry.local/v1/{endpoint}").respond( + 503, json={"error": {"message": "temporary outage", "type": "server_error"}} + ) + with pytest.raises(litellm.ServiceUnavailableError, match="temporary outage"): + await litellm.anthropic.messages.acreate( + model=model, + messages=[{"role": "user", "content": "hello"}], + max_tokens=16, + api_base="https://messages-bridge-retry.local/v1", + api_key="test-key", + max_retries=0, + **retry_kwargs, + ) + assert route.call_count == expected_attempts + + @pytest.mark.asyncio async def test_wrapper_async_replays_cached_converted_chat_stream_as_stream( monkeypatch: pytest.MonkeyPatch, From 57f6ba55995bbdde0d30d7ebe44dd70e12a08470 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 31 Oct 2026 11:25:31 -0700 Subject: [PATCH 4/5] fix(sdk): type the Decisions retry boundary --- litellm/utils.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 26ddc85b080..76c5268b694 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -307,7 +307,7 @@ except (ImportError, AttributeError, TypeError): # Convert to str (if necessary) claude_json_str = json.dumps(json_data) import importlib.metadata -from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized +from collections.abc import AsyncIterator, Awaitable, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from typing_extensions import assert_never @@ -2286,13 +2286,18 @@ def client(original_function): kwargs["num_retries"] = num_retries return litellm.responses_with_retries(*args, **kwargs) elif call_type in (CallTypes.decisions.value, CallTypes.systemone.value): - decision_num_retries, decision_kwargs = _get_wrapper_num_retries(kwargs=kwargs, exception=e) + decision_num_retries, decision_kwargs = _get_wrapper_num_retries( + kwargs=cast("dict[str, object]", kwargs), # cast-ok: decorator kwargs always have string keys + exception=e, + ) if ( decision_num_retries and not _is_litellm_router_call(decision_kwargs, is_async=False) and isinstance(e, openai.APIError) ): - return litellm.completion_with_retries( + return cast( # cast-ok: legacy retry helper forwards arbitrary SDK arguments + "Callable[..., object]", litellm.completion_with_retries + )( *args, **{**decision_kwargs, "num_retries": decision_num_retries, "original_function": wrapper}, ) @@ -2625,7 +2630,9 @@ def client(original_function): and not _is_litellm_router_call(kwargs, is_async=True) and isinstance(e, openai.APIError) ): - return await litellm.acompletion_with_retries( + return await cast( # cast-ok: legacy retry helper forwards arbitrary SDK arguments + "Callable[..., Awaitable[object]]", litellm.acompletion_with_retries + )( *args, **{ **kwargs, From b2d1ec68221d93c80a16a61a6927a9b32312e46e Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 31 Oct 2026 11:57:46 -0700 Subject: [PATCH 5/5] refactor(sdk): simplify Decisions retry calls --- litellm/main.py | 32 ++++++++++++++++++++------------ litellm/utils.py | 36 +++++++++++++----------------------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 3a97fc7d5e6..ce4bbf371f2 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6680,21 +6680,24 @@ def completion( ) -def completion_with_retries(*args, **kwargs): +def completion_with_retries( + *args: object, + num_retries: int = 3, + retry_strategy: RetryStrategy = "constant_retry", + original_function: Callable[..., object] | None = None, + **kwargs: object, +) -> object: """ - Executes a litellm.completion() with 3 retries + Retry completion or a supplied SDK call. """ try: import tenacity except Exception as e: raise Exception(f"tenacity import failed please run `pip install tenacity`. Error{e}") - num_retries: Final = kwargs.pop("num_retries", 3) # reset retries in .completion() kwargs["max_retries"] = 0 kwargs["num_retries"] = 0 - retry_strategy: Final[RetryStrategy] = kwargs.pop("retry_strategy", "constant_retry") - original_function: Final = kwargs.pop("original_function", completion) if retry_strategy == "exponential_backoff_retry": retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=10), @@ -6703,24 +6706,28 @@ def completion_with_retries(*args, **kwargs): ) else: retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) - return retryer(original_function, *args, **kwargs) + retry_target: Final[Callable[..., object]] = completion if original_function is None else original_function + return retryer(retry_target, *args, **kwargs) -async def acompletion_with_retries(*args, **kwargs): +async def acompletion_with_retries( + *args: object, + num_retries: int = 3, + retry_strategy: RetryStrategy = "constant_retry", + original_function: Callable[..., object] | None = None, + **kwargs: object, +) -> object: """ [DEPRECATED]. Use 'acompletion' or router.acompletion instead! - Executes a litellm.completion() with 3 retries + Retry completion or a supplied SDK call. """ try: import tenacity except Exception as e: raise Exception(f"tenacity import failed please run `pip install tenacity`. Error{e}") - num_retries: Final = kwargs.pop("num_retries", 3) kwargs["max_retries"] = 0 kwargs["num_retries"] = 0 - retry_strategy: Final[RetryStrategy] = kwargs.pop("retry_strategy", "constant_retry") - original_function: Final = kwargs.pop("original_function", completion) if retry_strategy == "exponential_backoff_retry": retryer = tenacity.AsyncRetrying( wait=tenacity.wait_exponential(multiplier=1, max=10), @@ -6729,7 +6736,8 @@ async def acompletion_with_retries(*args, **kwargs): ) else: retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) - return await retryer(original_function, *args, **kwargs) + retry_target: Final[Callable[..., object]] = completion if original_function is None else original_function + return await retryer(retry_target, *args, **kwargs) def responses_with_retries(*args, **kwargs): diff --git a/litellm/utils.py b/litellm/utils.py index 76c5268b694..1982ff74b64 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -307,7 +307,7 @@ except (ImportError, AttributeError, TypeError): # Convert to str (if necessary) claude_json_str = json.dumps(json_data) import importlib.metadata -from collections.abc import AsyncIterator, Awaitable, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized +from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence, Sized from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from typing_extensions import assert_never @@ -2286,20 +2286,15 @@ def client(original_function): kwargs["num_retries"] = num_retries return litellm.responses_with_retries(*args, **kwargs) elif call_type in (CallTypes.decisions.value, CallTypes.systemone.value): - decision_num_retries, decision_kwargs = _get_wrapper_num_retries( - kwargs=cast("dict[str, object]", kwargs), # cast-ok: decorator kwargs always have string keys - exception=e, - ) + retry_call: Final[partial[object]] = partial(wrapper, *args, **kwargs) + decision_num_retries, _ = _get_wrapper_num_retries(kwargs=retry_call.keywords, exception=e) if ( decision_num_retries - and not _is_litellm_router_call(decision_kwargs, is_async=False) + and not _is_litellm_router_call(retry_call.keywords, is_async=False) and isinstance(e, openai.APIError) ): - return cast( # cast-ok: legacy retry helper forwards arbitrary SDK arguments - "Callable[..., object]", litellm.completion_with_retries - )( - *args, - **{**decision_kwargs, "num_retries": decision_num_retries, "original_function": wrapper}, + return litellm.completion_with_retries( + original_function=retry_call, num_retries=decision_num_retries ) traceback_exception: Final = traceback.format_exc() end_time = datetime.datetime.now() @@ -2630,18 +2625,13 @@ def client(original_function): and not _is_litellm_router_call(kwargs, is_async=True) and isinstance(e, openai.APIError) ): - return await cast( # cast-ok: legacy retry helper forwards arbitrary SDK arguments - "Callable[..., Awaitable[object]]", litellm.acompletion_with_retries - )( - *args, - **{ - **kwargs, - "num_retries": num_retries, - "original_function": wrapper_async, - "retry_strategy": "exponential_backoff_retry" - if isinstance(e, openai.RateLimitError) - else "constant_retry", - }, + retry_call: Final[partial[object]] = partial(wrapper_async, *args, **kwargs) + return await litellm.acompletion_with_retries( + num_retries=num_retries, + original_function=retry_call, + retry_strategy="exponential_backoff_retry" + if isinstance(e, openai.RateLimitError) + else "constant_retry", ) deployment_num_retries: Final = kwargs.get("num_retries") if deployment_num_retries is not None: