From c0321991ee06373d3af85e5557deb144ecf7ab66 Mon Sep 17 00:00:00 2001 From: hphproton <44660214+hphproton@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:15:57 +0700 Subject: [PATCH] fix: keep litellm.num_retries after a failed call --- litellm/utils.py | 26 ++--------------------- tests/unit/test_utils.py | 46 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 66d78a03075..4bc95a07d15 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1972,18 +1972,7 @@ def client(original_function): return result except Exception as e: if call_type == CallTypes.completion.value: - num_retries = kwargs.get("num_retries", None) or litellm.num_retries or None - if kwargs.get("retry_policy", None): - get_num_retries_from_retry_policy: Callable[..., int | None] = ( - litellm_utils.get_num_retries_from_retry_policy - ) - reset_retry_policy = litellm_utils.reset_retry_policy - num_retries = get_num_retries_from_retry_policy( - exception=e, - retry_policy=kwargs.get("retry_policy"), - ) - kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops - litellm.num_retries = None # set retries to None to prevent infinite loops + num_retries, kwargs = _get_wrapper_num_retries(kwargs=kwargs, exception=e) context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) @@ -2009,16 +1998,7 @@ def client(original_function): kwargs["model"] = context_window_fallback_dict[model] return original_function(*args, **kwargs) elif call_type == CallTypes.responses.value: - num_retries = kwargs.get("num_retries", None) or litellm.num_retries or None - if kwargs.get("retry_policy", None): - get_num_retries_from_retry_policy = litellm_utils.get_num_retries_from_retry_policy - reset_retry_policy = litellm_utils.reset_retry_policy - num_retries = get_num_retries_from_retry_policy( - exception=e, - retry_policy=kwargs.get("retry_policy"), - ) - kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops - litellm.num_retries = None # set retries to None to prevent infinite loops + num_retries, kwargs = _get_wrapper_num_retries(kwargs=kwargs, exception=e) is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( @@ -2306,7 +2286,6 @@ def client(original_function): num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: - litellm.num_retries = None # set retries to None to prevent infinite loops kwargs["num_retries"] = num_retries kwargs["original_function"] = original_function if isinstance(e, openai.RateLimitError): # rate limiting specific error @@ -2337,7 +2316,6 @@ def client(original_function): num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: - litellm.num_retries = None # set retries to None to prevent infinite loops kwargs["num_retries"] = num_retries kwargs["original_function"] = original_function if isinstance(e, openai.RateLimitError): # rate limiting specific error diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 69ef2caf8f4..90a5434484b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10094,3 +10094,49 @@ def test_vertex_llama3_model_info_resolves_without_the_meta_prefix(): prefixed_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") assert litellm.get_model_info(model=short_name, custom_llm_provider="vertex_ai") == prefixed_info, model assert prefixed_info["key"] == f"vertex_ai/{model}", model + + +_FAILING_CALL_CASES: Final = ( + pytest.param("completion", "chat/completions", {"messages": [{"role": "user", "content": "hi"}]}, id="completion"), + pytest.param("responses", "responses", {"input": "hi"}, id="responses"), +) +_FAILING_ASYNC_CALL_CASES: Final = ( + pytest.param("acompletion", "chat/completions", {"messages": [{"role": "user", "content": "hi"}]}, id="acompletion"), + pytest.param("aresponses", "responses", {"input": "hi"}, id="aresponses"), +) + + +@respx.mock +@pytest.mark.parametrize("call_name, path, payload", _FAILING_CALL_CASES) +def test_failed_call_keeps_global_num_retries_for_later_calls( + monkeypatch: pytest.MonkeyPatch, call_name: str, path: str, payload: dict[str, object] +) -> None: + monkeypatch.setattr(litellm, "num_retries", 3) + route: Final = respx.post(f"https://api.openai.com/v1/{path}").respond(500, json={"error": {"message": "boom"}}) + call: Final = getattr(litellm, call_name) + + for _ in range(2): + with pytest.raises(litellm.InternalServerError): + call(model="openai/gpt-5.6", api_key="sk-test", max_retries=0, **payload) + + assert route.call_count == 8 + assert litellm.num_retries == 3 + + +@pytest.mark.asyncio +@respx.mock +@pytest.mark.parametrize("call_name, path, payload", _FAILING_ASYNC_CALL_CASES) +async def test_failed_async_call_keeps_global_num_retries_for_later_calls( + monkeypatch: pytest.MonkeyPatch, call_name: str, path: str, payload: dict[str, object] +) -> None: + monkeypatch.setattr(litellm, "num_retries", 3) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx.post(f"https://api.openai.com/v1/{path}").respond(500, json={"error": {"message": "boom"}}) + call: Final = getattr(litellm, call_name) + + for _ in range(2): + with pytest.raises(litellm.InternalServerError): + await call(model="openai/gpt-5.6", api_key="sk-test", max_retries=0, **payload) + + assert route.call_count == 8 + assert litellm.num_retries == 3