From a3e663fa033df101da4408607327249b5883e125 Mon Sep 17 00:00:00 2001 From: KJyothirmayi-2025sl93025 <2025sl93025@wilp.bits-pilani.ac.in> Date: Wed, 4 Nov 2026 10:47:59 +0530 Subject: [PATCH 1/7] fix(router): enforce caller deadline across retries and backoff sleeps Treats x-litellm-timeout (and x-stainless-timeout as fallback) as an end-to-end deadline for the entire router call, not just per-attempt. When a timeout header is present, a monotonic request_deadline is recorded in the request data. async_function_with_retries now checks the remaining budget before every backoff sleep: if the budget is already exhausted it raises immediately; otherwise it caps the sleep to the remaining time so the router never outlives the caller's deadline. Fixes #49751 --- litellm/proxy/_types.py | 1 + litellm/proxy/litellm_pre_call_utils.py | 3 +- litellm/router_python_core.py | 21 ++- tests/unit/test_router/test_router_timeout.py | 133 ++++++++++++++++++ 4 files changed, 154 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c9960d794c6..f648d059fda 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4535,6 +4535,7 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False): # cooldown_handlers._trigger_cooldown_for_failed_deployment). client_side_timeout: bool keepalive_seconds: float | None + request_deadline: float class LitellmMetadataFromRequestHeaders(TypedDict, total=False): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4444d6d25f9..52fb397a70e 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1657,7 +1657,7 @@ class LiteLLMProxyRequestSetup: }); ``` """ - timeout_header: Final = headers.get("x-litellm-timeout", None) + timeout_header: Final = headers.get("x-litellm-timeout") or headers.get("x-stainless-timeout") if timeout_header is not None: return float(timeout_header) return None @@ -1894,6 +1894,7 @@ class LiteLLMProxyRequestSetup: header_timeout: Final = LiteLLMProxyRequestSetup._get_timeout_from_request(headers) if header_timeout is not None: data["timeout"] = header_timeout + data["request_deadline"] = time.monotonic() + header_timeout header_stream_timeout: Final = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers) if header_stream_timeout is not None: diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index ee67d82abfa..a0afcd0160c 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -9552,6 +9552,9 @@ class PythonCore(DeprecatedRouterMembers): if model_list is not None: _metadata.update({"model_group_size": len(model_list)}) + _deadline_raw: Final = kwargs.get("request_deadline") + request_deadline: Final[float | None] = _deadline_raw if isinstance(_deadline_raw, float) else None + verbose_router_logger.debug( "async function w/ retries: original_function - %s, num_retries - %s", original_function, num_retries ) @@ -9653,7 +9656,13 @@ class PythonCore(DeprecatedRouterMembers): all_deployments=_all_deployments, ) - await self._host.sleep(retry_after) + if request_deadline is not None: + _remaining: Final = request_deadline - time.monotonic() + if _remaining <= 0: + raise original_exception + await self._host.sleep(min(retry_after, _remaining)) + else: + await self._host.sleep(retry_after) for current_attempt in range(num_retries): try: @@ -9723,14 +9732,20 @@ class PythonCore(DeprecatedRouterMembers): _DEPLOYMENTS_ADAPTER.validate_python(_healthy_deployments), skipped_ids ): raise - _timeout = self._time_to_sleep_before_retry( + _sleep_duration: Final = self._time_to_sleep_before_retry( e=e, remaining_retries=remaining_retries, num_retries=num_retries, healthy_deployments=_healthy_deployments, all_deployments=_all_deployments, ) - await self._host.sleep(_timeout) + if request_deadline is not None: + _loop_remaining: Final = request_deadline - time.monotonic() + if _loop_remaining <= 0: + raise e + await self._host.sleep(min(_sleep_duration, _loop_remaining)) + else: + await self._host.sleep(_sleep_duration) if type(original_exception) in self._host.dynamic().exception_types: setattr(original_exception, "max_retries", num_retries) diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index 8fc32620f71..b889aaf1ac1 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -690,4 +690,137 @@ def test_azure_router_timeout_raises_after_httpx_read_timeout( messages=[{"role": "user", "content": "timeout contract"}], ) + +def test_request_deadline_set_when_litellm_timeout_header_present() -> None: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy._types import UserAPIKeyAuth + + before: Final = time.monotonic() + result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={"x-litellm-timeout": "10"}, + request_data={}, + user_api_key_dict=UserAPIKeyAuth(), + ) + after: Final = time.monotonic() + + assert result.get("timeout") == 10.0 + deadline: Final = result.get("request_deadline") + assert isinstance(deadline, float) + assert before + 10.0 <= deadline <= after + 10.0 + + +def test_request_deadline_set_when_stainless_timeout_header_present() -> None: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy._types import UserAPIKeyAuth + + before: Final = time.monotonic() + result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={"x-stainless-timeout": "15"}, + request_data={}, + user_api_key_dict=UserAPIKeyAuth(), + ) + after: Final = time.monotonic() + + assert result.get("timeout") == 15.0 + deadline: Final = result.get("request_deadline") + assert isinstance(deadline, float) + assert before + 15.0 <= deadline <= after + 15.0 + + +def test_request_deadline_not_set_when_no_timeout_header() -> None: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy._types import UserAPIKeyAuth + + result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={}, + request_data={}, + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert result.get("request_deadline") is None + + +@pytest.mark.asyncio +async def test_retry_skips_sleep_when_deadline_already_expired() -> None: + sleep_calls: Final[list[float]] = [] + + async def fake_sleep(duration: float) -> None: + sleep_calls.append(duration) + + router: Final = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}, + } + ], + num_retries=3, + ) + + error: Final = litellm.RateLimitError( + message="rate limited", model="gpt-3.5-turbo", llm_provider="openai" + ) + error.litellm_response_headers = httpx.Headers({"retry-after": "60"}) + + with patch.object(router, "_host") as mock_host: + mock_host.sleep = fake_sleep + mock_host.invoke = router._host.invoke # type: ignore[attr-defined] + mock_host.dynamic = router._host.dynamic # type: ignore[attr-defined] + + with pytest.raises(litellm.RateLimitError): + await router.async_function_with_retries( + original_function=AsyncMock(side_effect=error), + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + request_deadline=time.monotonic() - 1.0, + fallbacks=None, + context_window_fallbacks=None, + content_policy_fallbacks=None, + ) + + assert sleep_calls == [], f"expected no sleeps when deadline already past, got {sleep_calls}" + + +@pytest.mark.asyncio +async def test_retry_caps_sleep_to_remaining_budget() -> None: + sleep_calls: Final[list[float]] = [] + + async def fake_sleep(duration: float) -> None: + sleep_calls.append(duration) + + router: Final = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}, + } + ], + num_retries=1, + ) + + error: Final = litellm.RateLimitError( + message="rate limited", model="gpt-3.5-turbo", llm_provider="openai" + ) + error.litellm_response_headers = httpx.Headers({"retry-after": "60"}) + + deadline: Final = time.monotonic() + 2.0 + + with patch.object(router, "_host") as mock_host: + mock_host.sleep = fake_sleep + mock_host.invoke = router._host.invoke # type: ignore[attr-defined] + mock_host.dynamic = router._host.dynamic # type: ignore[attr-defined] + + with pytest.raises(litellm.RateLimitError): + await router.async_function_with_retries( + original_function=AsyncMock(side_effect=error), + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + request_deadline=deadline, + fallbacks=None, + context_window_fallbacks=None, + content_policy_fallbacks=None, + ) + + assert all(s <= 2.5 for s in sleep_calls), f"sleep not capped to budget: {sleep_calls}" + assert route.call_count == 1 From 5374002fd756d54b59d71b439b257d527842c6cd Mon Sep 17 00:00:00 2001 From: KJyothirmayi-2025sl93025 <2025sl93025@wilp.bits-pilani.ac.in> Date: Wed, 4 Nov 2026 10:47:59 +0530 Subject: [PATCH 2/7] test(router): fix mock pattern and add priority test for deadline enforcement Replace patch.object(router, "_host") with patch.object(router._core._host, "sleep", new=fake_sleep) so only the sleep method is intercepted and invoke/dynamic continue to work normally. Remove stale route.call_count assertion that referenced an out-of-scope variable. Add test_litellm_timeout_takes_priority_over_stainless_timeout to cover header precedence. --- tests/unit/test_router/test_router_timeout.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index b889aaf1ac1..d773303457c 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -762,11 +762,7 @@ async def test_retry_skips_sleep_when_deadline_already_expired() -> None: ) error.litellm_response_headers = httpx.Headers({"retry-after": "60"}) - with patch.object(router, "_host") as mock_host: - mock_host.sleep = fake_sleep - mock_host.invoke = router._host.invoke # type: ignore[attr-defined] - mock_host.dynamic = router._host.dynamic # type: ignore[attr-defined] - + with patch.object(router._host, "sleep", new=fake_sleep): # type: ignore[attr-defined] with pytest.raises(litellm.RateLimitError): await router.async_function_with_retries( original_function=AsyncMock(side_effect=error), @@ -805,11 +801,7 @@ async def test_retry_caps_sleep_to_remaining_budget() -> None: deadline: Final = time.monotonic() + 2.0 - with patch.object(router, "_host") as mock_host: - mock_host.sleep = fake_sleep - mock_host.invoke = router._host.invoke # type: ignore[attr-defined] - mock_host.dynamic = router._host.dynamic # type: ignore[attr-defined] - + with patch.object(router._host, "sleep", new=fake_sleep): # type: ignore[attr-defined] with pytest.raises(litellm.RateLimitError): await router.async_function_with_retries( original_function=AsyncMock(side_effect=error), @@ -823,4 +815,15 @@ async def test_retry_caps_sleep_to_remaining_budget() -> None: assert all(s <= 2.5 for s in sleep_calls), f"sleep not capped to budget: {sleep_calls}" - assert route.call_count == 1 + +def test_litellm_timeout_takes_priority_over_stainless_timeout() -> None: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy._types import UserAPIKeyAuth + + result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={"x-litellm-timeout": "5", "x-stainless-timeout": "30"}, + request_data={}, + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert result.get("timeout") == 5.0, "x-litellm-timeout must win when both headers present" From a5c339168d560dee98017838006115b13a54856b Mon Sep 17 00:00:00 2001 From: KJyothirmayi-2025sl93025 <2025sl93025@wilp.bits-pilani.ac.in> Date: Wed, 4 Nov 2026 10:47:59 +0530 Subject: [PATCH 3/7] test(router): fix ruff violations in deadline test additions Sort imports alphabetically within test functions and merge the two nested with statements into a single parenthesized form (SIM117). --- tests/unit/test_router/test_router_timeout.py | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index d773303457c..f0647dc7556 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -692,8 +692,8 @@ def test_azure_router_timeout_raises_after_httpx_read_timeout( def test_request_deadline_set_when_litellm_timeout_header_present() -> None: - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup before: Final = time.monotonic() result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( @@ -710,8 +710,8 @@ def test_request_deadline_set_when_litellm_timeout_header_present() -> None: def test_request_deadline_set_when_stainless_timeout_header_present() -> None: - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup before: Final = time.monotonic() result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( @@ -728,8 +728,8 @@ def test_request_deadline_set_when_stainless_timeout_header_present() -> None: def test_request_deadline_not_set_when_no_timeout_header() -> None: - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( headers={}, @@ -762,17 +762,19 @@ async def test_retry_skips_sleep_when_deadline_already_expired() -> None: ) error.litellm_response_headers = httpx.Headers({"retry-after": "60"}) - with patch.object(router._host, "sleep", new=fake_sleep): # type: ignore[attr-defined] - with pytest.raises(litellm.RateLimitError): - await router.async_function_with_retries( - original_function=AsyncMock(side_effect=error), - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "hi"}], - request_deadline=time.monotonic() - 1.0, - fallbacks=None, - context_window_fallbacks=None, - content_policy_fallbacks=None, - ) + with ( + patch.object(router._host, "sleep", new=fake_sleep), # type: ignore[attr-defined] + pytest.raises(litellm.RateLimitError), + ): + await router.async_function_with_retries( + original_function=AsyncMock(side_effect=error), + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + request_deadline=time.monotonic() - 1.0, + fallbacks=None, + context_window_fallbacks=None, + content_policy_fallbacks=None, + ) assert sleep_calls == [], f"expected no sleeps when deadline already past, got {sleep_calls}" @@ -801,24 +803,26 @@ async def test_retry_caps_sleep_to_remaining_budget() -> None: deadline: Final = time.monotonic() + 2.0 - with patch.object(router._host, "sleep", new=fake_sleep): # type: ignore[attr-defined] - with pytest.raises(litellm.RateLimitError): - await router.async_function_with_retries( - original_function=AsyncMock(side_effect=error), - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "hi"}], - request_deadline=deadline, - fallbacks=None, - context_window_fallbacks=None, - content_policy_fallbacks=None, - ) + with ( + patch.object(router._host, "sleep", new=fake_sleep), # type: ignore[attr-defined] + pytest.raises(litellm.RateLimitError), + ): + await router.async_function_with_retries( + original_function=AsyncMock(side_effect=error), + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + request_deadline=deadline, + fallbacks=None, + context_window_fallbacks=None, + content_policy_fallbacks=None, + ) assert all(s <= 2.5 for s in sleep_calls), f"sleep not capped to budget: {sleep_calls}" def test_litellm_timeout_takes_priority_over_stainless_timeout() -> None: - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( headers={"x-litellm-timeout": "5", "x-stainless-timeout": "30"}, From 3d9adc3229c5bcb3c649db1cfb02626b61aae367 Mon Sep 17 00:00:00 2001 From: KJyothirmayi-2025sl93025 <2025sl93025@wilp.bits-pilani.ac.in> Date: Wed, 4 Nov 2026 18:42:48 +0530 Subject: [PATCH 4/7] fix(router): add pre-call deadline check after backoff and fix test patching Add a deadline check immediately before each provider attempt in the retry loop so that a sleep that consumes all remaining budget cannot be followed by a provider call. Also add early bail in both backoff paths when the sleep duration >= remaining budget (avoids a useless sleep before the inevitable rejection). Fix the sleep-patching pattern in deadline tests: PythonHost uses __slots__ which prevents instance-level attribute assignment, so patch the class method instead. Restore the accidentally-dropped test_litellm_timeout_takes_priority_over_stainless_timeout function header, and add metadata kwarg required by _record_failed_attempt. Three new tests assert the exact invariant the reviewer asked for: zero additional provider calls when the first-backoff sleep exhausts the deadline, one retry when the in-loop sleep nearly exhausts it, and a positive control confirming retries do proceed with sufficient budget. --- litellm/router_python_core.py | 6 +- tests/unit/test_router/test_router_timeout.py | 146 +++++++++++++++++- 2 files changed, 146 insertions(+), 6 deletions(-) diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index a0afcd0160c..744a32a45f7 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -9658,13 +9658,15 @@ class PythonCore(DeprecatedRouterMembers): if request_deadline is not None: _remaining: Final = request_deadline - time.monotonic() - if _remaining <= 0: + if _remaining <= 0 or retry_after >= _remaining: raise original_exception await self._host.sleep(min(retry_after, _remaining)) else: await self._host.sleep(retry_after) for current_attempt in range(num_retries): + if request_deadline is not None and request_deadline - time.monotonic() <= 0: + raise original_exception try: # Update retry tracking metadata before each retry attempt _metadata["attempted_retries"] = current_attempt + 1 @@ -9741,7 +9743,7 @@ class PythonCore(DeprecatedRouterMembers): ) if request_deadline is not None: _loop_remaining: Final = request_deadline - time.monotonic() - if _loop_remaining <= 0: + if _loop_remaining <= 0 or _sleep_duration >= _loop_remaining: raise e await self._host.sleep(min(_sleep_duration, _loop_remaining)) else: diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index f0647dc7556..b01d924d170 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -15,6 +15,7 @@ import respx import litellm from litellm import Router +from litellm.router_python_host import PythonHost from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.router_utils.cooldown_handlers import is_caller_timeout_408 @@ -744,7 +745,7 @@ def test_request_deadline_not_set_when_no_timeout_header() -> None: async def test_retry_skips_sleep_when_deadline_already_expired() -> None: sleep_calls: Final[list[float]] = [] - async def fake_sleep(duration: float) -> None: + async def fake_sleep(_host: object, duration: float) -> None: sleep_calls.append(duration) router: Final = Router( @@ -763,13 +764,14 @@ async def test_retry_skips_sleep_when_deadline_already_expired() -> None: error.litellm_response_headers = httpx.Headers({"retry-after": "60"}) with ( - patch.object(router._host, "sleep", new=fake_sleep), # type: ignore[attr-defined] + patch.object(PythonHost, "sleep", new=fake_sleep), pytest.raises(litellm.RateLimitError), ): await router.async_function_with_retries( original_function=AsyncMock(side_effect=error), model="gpt-3.5-turbo", messages=[{"role": "user", "content": "hi"}], + metadata={"model_group": "gpt-3.5-turbo"}, request_deadline=time.monotonic() - 1.0, fallbacks=None, context_window_fallbacks=None, @@ -783,7 +785,7 @@ async def test_retry_skips_sleep_when_deadline_already_expired() -> None: async def test_retry_caps_sleep_to_remaining_budget() -> None: sleep_calls: Final[list[float]] = [] - async def fake_sleep(duration: float) -> None: + async def fake_sleep(_host: object, duration: float) -> None: sleep_calls.append(duration) router: Final = Router( @@ -804,13 +806,14 @@ async def test_retry_caps_sleep_to_remaining_budget() -> None: deadline: Final = time.monotonic() + 2.0 with ( - patch.object(router._host, "sleep", new=fake_sleep), # type: ignore[attr-defined] + patch.object(PythonHost, "sleep", new=fake_sleep), pytest.raises(litellm.RateLimitError), ): await router.async_function_with_retries( original_function=AsyncMock(side_effect=error), model="gpt-3.5-turbo", messages=[{"role": "user", "content": "hi"}], + metadata={"model_group": "gpt-3.5-turbo"}, request_deadline=deadline, fallbacks=None, context_window_fallbacks=None, @@ -820,6 +823,141 @@ async def test_retry_caps_sleep_to_remaining_budget() -> None: assert all(s <= 2.5 for s in sleep_calls), f"sleep not capped to budget: {sleep_calls}" +@pytest.mark.asyncio +async def test_no_provider_call_after_first_backoff_exhausts_deadline() -> None: + """Sleep that runs to the deadline must not be followed by a provider call.""" + fake_now: Final[list[float]] = [0.0] + deadline: Final = 5.0 + call_count: Final[list[int]] = [0] + + rate_error: Final = litellm.RateLimitError( + message="rate limited", model="gpt-3.5-turbo", llm_provider="openai" + ) + rate_error.litellm_response_headers = httpx.Headers({"retry-after": "1"}) + + async def original(*args: object, **kwargs: object) -> object: + call_count[0] += 1 + raise rate_error + + async def sleep_to_deadline(_host: object, _duration: float) -> None: + fake_now[0] = deadline + + router: Final = Router( + model_list=[ + {"model_name": "gpt-3.5-turbo", "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}} + ], + num_retries=3, + ) + + with ( + patch.object(time, "monotonic", lambda: fake_now[0]), + patch.object(PythonHost, "sleep", new=sleep_to_deadline), + pytest.raises(litellm.RateLimitError), + ): + await router.async_function_with_retries( + original_function=original, + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + metadata={"model_group": "gpt-3.5-turbo"}, + request_deadline=deadline, + fallbacks=None, + context_window_fallbacks=None, + content_policy_fallbacks=None, + ) + + assert call_count[0] == 1, f"expected 1 call (initial only), got {call_count[0]}" + + +@pytest.mark.asyncio +async def test_no_provider_call_after_in_loop_sleep_exhausts_deadline() -> None: + """After an in-loop backoff, if the remaining budget < sleep duration, no further call starts.""" + fake_now: Final[list[float]] = [0.0] + deadline: Final = 5.0 + call_count: Final[list[int]] = [0] + + rate_error: Final = litellm.RateLimitError( + message="rate limited", model="gpt-3.5-turbo", llm_provider="openai" + ) + rate_error.litellm_response_headers = httpx.Headers({"retry-after": "1"}) + + async def original(*args: object, **kwargs: object) -> object: + call_count[0] += 1 + raise rate_error + + async def fake_sleep(_host: object, _duration: float) -> None: + fake_now[0] = deadline - 0.001 + + router: Final = Router( + model_list=[ + {"model_name": "gpt-3.5-turbo", "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}} + ], + num_retries=3, + ) + + with ( + patch.object(time, "monotonic", lambda: fake_now[0]), + patch.object(PythonHost, "sleep", new=fake_sleep), + pytest.raises(litellm.RateLimitError), + ): + await router.async_function_with_retries( + original_function=original, + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + metadata={"model_group": "gpt-3.5-turbo"}, + request_deadline=deadline, + fallbacks=None, + context_window_fallbacks=None, + content_policy_fallbacks=None, + ) + + assert call_count[0] == 2, f"expected 2 calls (initial + one retry), got {call_count[0]}" + + +@pytest.mark.asyncio +async def test_retry_proceeds_when_sufficient_budget_remains() -> None: + """A retry attempt is made when remaining budget outlasts the backoff sleep.""" + fake_now: Final[list[float]] = [0.0] + deadline: Final = 5.0 + call_count: Final[list[int]] = [0] + + rate_error: Final = litellm.RateLimitError( + message="rate limited", model="gpt-3.5-turbo", llm_provider="openai" + ) + rate_error.litellm_response_headers = httpx.Headers({"retry-after": "1"}) + + async def original(*args: object, **kwargs: object) -> object: + call_count[0] += 1 + raise rate_error + + async def fake_sleep(_host: object, _duration: float) -> None: + fake_now[0] += 0.1 + + router: Final = Router( + model_list=[ + {"model_name": "gpt-3.5-turbo", "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}} + ], + num_retries=1, + ) + + with ( + patch.object(time, "monotonic", lambda: fake_now[0]), + patch.object(PythonHost, "sleep", new=fake_sleep), + pytest.raises(litellm.RateLimitError), + ): + await router.async_function_with_retries( + original_function=original, + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + metadata={"model_group": "gpt-3.5-turbo"}, + request_deadline=deadline, + fallbacks=None, + context_window_fallbacks=None, + content_policy_fallbacks=None, + ) + + assert call_count[0] == 2, f"expected 2 calls (initial + one retry), got {call_count[0]}" + + def test_litellm_timeout_takes_priority_over_stainless_timeout() -> None: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup From c0a1c9c4b451192fa4cbb0e865066c899394ca7a Mon Sep 17 00:00:00 2001 From: KJyothirmayi-2025sl93025 <2025sl93025@wilp.bits-pilani.ac.in> Date: Wed, 4 Nov 2026 19:28:33 +0530 Subject: [PATCH 5/7] fix(router): clamp per-attempt timeout to remaining deadline budget _get_non_stream_timeout and _get_stream_timeout now cap the resolved timeout to request_deadline - monotonic() when a deadline is present. This closes the residual gap the reviewer called "attempt-timeout bound": a provider call that starts just before the deadline can no longer run for its full configured timeout (e.g. 30 s) past the caller's deadline. Two new unit tests cover the invariant directly via _get_non_stream_timeout: one where configured > remaining (result clamped to remaining) and one where configured < remaining (configured wins unchanged). --- litellm/router_python_core.py | 38 +++++++++++++------ tests/unit/test_router/test_router_timeout.py | 32 ++++++++++++++++ 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 744a32a45f7..13c4b6721ff 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -5560,29 +5560,43 @@ class PythonCore(DeprecatedRouterMembers): self, kwargs: Mapping[str, object], data: Mapping[str, object] | GenericLiteLLMParams ) -> object: """Helper to get stream timeout from kwargs or deployment params""" - return ( - kwargs.get("stream_timeout", None) # the params dynamically set by user + configured: Final = ( + kwargs.get("stream_timeout", None) or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier") - or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment - or self.stream_timeout # timeout set on router - or self.request_timeout # litellm_settings.request_timeout (per-attempt) + or data.get("stream_timeout", None) + or self.stream_timeout + or self.request_timeout or self._core_default_litellm_params.get("stream_timeout", None) ) + _deadline: Final = kwargs.get("request_deadline") + if not isinstance(_deadline, float): + return configured + _remaining: Final = _deadline - time.monotonic() + if not isinstance(configured, (int, float)): + return _remaining + return min(float(configured), _remaining) def _get_non_stream_timeout( self, kwargs: Mapping[str, object], data: Mapping[str, object] | GenericLiteLLMParams ) -> object: """Helper to get non-stream timeout from kwargs or deployment params""" - return ( - kwargs.get("timeout", None) # the params dynamically set by user - or kwargs.get("request_timeout", None) # the params dynamically set by user + configured: Final = ( + kwargs.get("timeout", None) + or kwargs.get("request_timeout", None) or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier") - or data.get("timeout", None) # timeout set on litellm_params for this deployment - or data.get("request_timeout", None) # timeout set on litellm_params for this deployment - or self.request_timeout # litellm_settings.request_timeout (per-attempt) - or self.timeout # timeout set on router (router_settings.timeout) + or data.get("timeout", None) + or data.get("request_timeout", None) + or self.request_timeout + or self.timeout or self._core_default_litellm_params.get("timeout", None) ) + _deadline: Final = kwargs.get("request_deadline") + if not isinstance(_deadline, float): + return configured + _remaining: Final = _deadline - time.monotonic() + if not isinstance(configured, (int, float)): + return _remaining + return min(float(configured), _remaining) def _get_timeout(self, kwargs: Mapping[str, object], data: Mapping[str, object] | GenericLiteLLMParams) -> object: """Helper to get timeout from kwargs or deployment params""" diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index b01d924d170..2f6ee71bb74 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -969,3 +969,35 @@ def test_litellm_timeout_takes_priority_over_stainless_timeout() -> None: ) assert result.get("timeout") == 5.0, "x-litellm-timeout must win when both headers present" + + +def test_per_attempt_timeout_clamped_to_deadline_remaining() -> None: + """When configured timeout > remaining budget, _get_non_stream_timeout must clamp to remaining.""" + fake_now: Final[list[float]] = [0.0] + router: Final = Router( + model_list=[ + {"model_name": "gpt-3.5-turbo", "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}} + ] + ) + with patch.object(time, "monotonic", lambda: fake_now[0]): + result: Final = router._get_non_stream_timeout( + kwargs={"timeout": 30.0, "request_deadline": 2.0}, + data={}, + ) + assert result == 2.0, f"expected timeout clamped to 2.0 (remaining budget), got {result}" + + +def test_per_attempt_timeout_not_extended_beyond_configured() -> None: + """When configured timeout < remaining budget, _get_non_stream_timeout must return configured.""" + fake_now: Final[list[float]] = [0.0] + router: Final = Router( + model_list=[ + {"model_name": "gpt-3.5-turbo", "litellm_params": {"model": "gpt-3.5-turbo", "api_key": "sk-test"}} + ] + ) + with patch.object(time, "monotonic", lambda: fake_now[0]): + result: Final = router._get_non_stream_timeout( + kwargs={"timeout": 1.0, "request_deadline": 10.0}, + data={}, + ) + assert result == 1.0, f"expected configured timeout 1.0 to win over deadline remaining 10.0, got {result}" From ad776e21a0e5d0e7fc4d15fc9e8044eb42a03fa5 Mon Sep 17 00:00:00 2001 From: KJyothirmayi-2025sl93025 <2025sl93025@wilp.bits-pilani.ac.in> Date: Wed, 4 Nov 2026 19:35:41 +0530 Subject: [PATCH 6/7] fix(lint): fix TRY201 bare-raise and LIT012 ReadOnly on request_deadline raise e -> raise (bare) in except block to satisfy TRY201. ReadOnly[float] on request_deadline field in LitellmDataForBackendLLMCall to satisfy LIT012 (all TypedDict fields must use ReadOnly qualifier). --- litellm/proxy/_types.py | 2 +- litellm/router_python_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f648d059fda..3f274a14ea7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4535,7 +4535,7 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False): # cooldown_handlers._trigger_cooldown_for_failed_deployment). client_side_timeout: bool keepalive_seconds: float | None - request_deadline: float + request_deadline: ReadOnly[float] class LitellmMetadataFromRequestHeaders(TypedDict, total=False): diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 13c4b6721ff..52846614035 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -9758,7 +9758,7 @@ class PythonCore(DeprecatedRouterMembers): if request_deadline is not None: _loop_remaining: Final = request_deadline - time.monotonic() if _loop_remaining <= 0 or _sleep_duration >= _loop_remaining: - raise e + raise await self._host.sleep(min(_sleep_duration, _loop_remaining)) else: await self._host.sleep(_sleep_duration) From 95b3bf4d6b3c9cb92b9f4615b00c8b67328a3fb3 Mon Sep 17 00:00:00 2001 From: KJyothirmayi-2025sl93025 <2025sl93025@wilp.bits-pilani.ac.in> Date: Wed, 4 Nov 2026 19:57:42 +0530 Subject: [PATCH 7/7] fix(proxy): set request_deadline from body timeout, not only from headers add_litellm_data_for_backend_llm_call only created request_deadline when a timeout header was present. A body-only request_data={"timeout": 10} set client_side_timeout=True but left request_deadline unset, so the retry loop and per-attempt timeout clamp had nothing to enforce. Now the effective deadline is built from header_timeout if present, else from the body timeout field. Header still takes precedence when both exist. Three new tests cover: body-only construction, header-wins-over-body, and the existing no-timeout path remains deadline-free. --- litellm/proxy/litellm_pre_call_utils.py | 12 ++++++- tests/unit/test_router/test_router_timeout.py | 33 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 52fb397a70e..795636d4ac6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1894,7 +1894,17 @@ class LiteLLMProxyRequestSetup: header_timeout: Final = LiteLLMProxyRequestSetup._get_timeout_from_request(headers) if header_timeout is not None: data["timeout"] = header_timeout - data["request_deadline"] = time.monotonic() + header_timeout + + _body_timeout: Final = request_data.get("timeout") + _deadline_secs: Final = ( + header_timeout + if header_timeout is not None + else float(_body_timeout) + if isinstance(_body_timeout, (int, float)) + else None + ) + if _deadline_secs is not None: + data["request_deadline"] = time.monotonic() + _deadline_secs header_stream_timeout: Final = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers) if header_stream_timeout is not None: diff --git a/tests/unit/test_router/test_router_timeout.py b/tests/unit/test_router/test_router_timeout.py index 2f6ee71bb74..986e22ef303 100644 --- a/tests/unit/test_router/test_router_timeout.py +++ b/tests/unit/test_router/test_router_timeout.py @@ -741,6 +741,39 @@ def test_request_deadline_not_set_when_no_timeout_header() -> None: assert result.get("request_deadline") is None +def test_request_deadline_set_from_body_timeout() -> None: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + fake_now: Final[list[float]] = [0.0] + with patch.object(time, "monotonic", lambda: fake_now[0]): + result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={}, + request_data={"timeout": 10}, + user_api_key_dict=UserAPIKeyAuth(), + ) + + deadline: Final = result.get("request_deadline") + assert isinstance(deadline, float), "request_deadline must be set when body timeout is present" + assert deadline == 10.0, f"expected deadline 10.0 (now=0 + 10), got {deadline}" + + +def test_request_deadline_body_timeout_header_wins() -> None: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + fake_now: Final[list[float]] = [0.0] + with patch.object(time, "monotonic", lambda: fake_now[0]): + result: Final = LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call( + headers={"x-litellm-timeout": "5"}, + request_data={"timeout": 30}, + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert result.get("timeout") == 5.0, "header timeout must win in data[timeout]" + assert result.get("request_deadline") == 5.0, "deadline must use header timeout, not body" + + @pytest.mark.asyncio async def test_retry_skips_sleep_when_deadline_already_expired() -> None: sleep_calls: Final[list[float]] = []