From 163c0f3aee99ba61f12317453cd76741b9f1558b Mon Sep 17 00:00:00 2001 From: clonylu Date: Tue, 15 Sep 2026 15:49:06 +0800 Subject: [PATCH 1/7] fix(router): honor stream_timeout on the SDK-native passthrough route Anthropic /v1/messages and Bedrock /converse resolve their upstream timeout through resolve_llm_passthrough_timeout, which only reads timeout / request_timeout and then falls back to the 600s pass_through default. A stream_timeout set on the deployment or in router_settings was never consulted on that route, while /chat/completions honors it through Router._get_stream_timeout. For a streaming call the resolver now checks stream_timeout at each level before the non-stream key (kwargs -> litellm_params -> router), mirroring _get_stream_timeout; non-streaming resolution is unchanged. The router passes its stream_timeout alongside the explicit timeout. --- litellm/passthrough/timeout_utils.py | 23 ++++++-- litellm/router.py | 4 ++ .../test_pass_through_endpoints.py | 58 +++++++++++++++++++ tests/test_litellm/test_router.py | 58 +++++++++++++++++++ 4 files changed, 139 insertions(+), 4 deletions(-) diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index 39127d19183..fb649a9eeaf 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -34,22 +34,37 @@ def resolve_llm_passthrough_timeout( kwargs: dict | None = None, litellm_params: dict | None = None, router_timeout: float | None = None, + router_stream_timeout: float | None = None, ) -> float: """ - Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse). + Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse, + Anthropic /v1/messages). - Precedence: kwargs timeout/request_timeout -> litellm_params timeout/request_timeout - -> router_timeout -> general_settings.pass_through_request_timeout -> 600s default. + Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params + timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout + -> 600s default. + + Streaming (``kwargs["stream"]`` truthy) additionally consults ``stream_timeout`` at each + level before the non-streaming key, matching ``Router._get_stream_timeout`` on the + completion route: kwargs stream_timeout -> kwargs timeout/request_timeout -> + litellm_params stream_timeout -> litellm_params timeout/request_timeout -> + router_stream_timeout -> router_timeout -> pass_through_request_timeout -> 600s. """ kwargs = kwargs or {} litellm_params = litellm_params or {} + is_stream: Final[bool] = bool(kwargs.get("stream", False)) + keys: Final[tuple[str, ...]] = ( + ("stream_timeout", "timeout", "request_timeout") if is_stream else ("timeout", "request_timeout") + ) for source in (kwargs, litellm_params): - for key in ("timeout", "request_timeout"): + for key in keys: val = source.get(key) if val is not None: return float(val) + if is_stream and router_stream_timeout is not None: + return float(router_stream_timeout) if router_timeout is not None: return float(router_timeout) diff --git a/litellm/router.py b/litellm/router.py index 1665583386f..7ce7ba30502 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3879,10 +3879,14 @@ class Router: _router_timeout: Final = ( float(self._explicit_timeout) if isinstance(self._explicit_timeout, (int, float)) else None ) + _router_stream_timeout: Final = ( + float(self.stream_timeout) if isinstance(self.stream_timeout, (int, float)) else None + ) kwargs["timeout"] = resolve_llm_passthrough_timeout( kwargs=kwargs, litellm_params=deployment["litellm_params"], router_timeout=_router_timeout, + router_stream_timeout=_router_stream_timeout, ) else: kwargs["timeout"] = self._get_timeout(kwargs=kwargs, data=deployment["litellm_params"]) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index d57bed430c1..d697a114613 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1119,6 +1119,64 @@ def test_resolve_llm_passthrough_timeout_precedence(): assert resolve_llm_passthrough_timeout() == 6.0 +def test_resolve_llm_passthrough_timeout_stream_timeout_precedence(): + # streaming: stream_timeout wins at each level, then falls through to the non-stream keys + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True, "stream_timeout": 1800, "timeout": 45}, + ) + == 1800.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + litellm_params={"stream_timeout": 1800, "timeout": 90}, + ) + == 1800.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + litellm_params={"timeout": 90}, + router_stream_timeout=1800, + ) + == 90.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + router_timeout=120, + router_stream_timeout=1800, + ) + == 1800.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + router_timeout=120, + ) + == 120.0 + ) + + # non-streaming: stream_timeout is ignored everywhere + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": False, "stream_timeout": 1800}, + litellm_params={"stream_timeout": 1800, "timeout": 90}, + router_stream_timeout=1800, + ) + == 90.0 + ) + with patch("litellm.proxy.proxy_server.general_settings", {}): + assert ( + resolve_llm_passthrough_timeout( + litellm_params={"stream_timeout": 1800}, + router_stream_timeout=1800, + ) + == DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS + ) + + @pytest.mark.asyncio async def test_pass_through_request_uses_resolved_timeout(): with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 3cabcd71627..70ce182c028 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5480,6 +5480,64 @@ def test_update_kwargs_with_deployment_uses_pass_through_request_timeout(): assert kwargs["timeout"] == 6.0 +def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): + """ + The SDK-native passthrough route (anthropic /v1/messages, bedrock /converse) resolves + its upstream timeout separately from the completion route. A streaming call must get + stream_timeout (deployment litellm_params first, then router_settings), while a + non-streaming call on the same deployment keeps the non-stream resolution. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "anthropic-with-stream-timeout", + "litellm_params": { + "model": "anthropic/claude-sonnet-4-5", + "api_key": "fake-key", + "stream_timeout": 1800, + }, + }, + { + "model_name": "anthropic-router-default", + "litellm_params": { + "model": "anthropic/claude-sonnet-4-5", + "api_key": "fake-key", + }, + }, + ], + stream_timeout=900, + ) + per_deployment, router_default = router.model_list + + with patch( + "litellm.proxy.proxy_server.general_settings", + {"pass_through_request_timeout": 6}, + ): + kwargs: dict = {"stream": True} + router._update_kwargs_with_deployment( + deployment=per_deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 1800.0 + + kwargs = {"stream": True} + router._update_kwargs_with_deployment( + deployment=router_default, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 900.0 + + kwargs = {"stream": False} + router._update_kwargs_with_deployment( + deployment=per_deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 6.0 + + @pytest.mark.asyncio async def test_router_acompletion_with_unknown_model_and_default_fallback(): """ From efb2bcd87fae4c6a78bb562cbcde98778b967a79 Mon Sep 17 00:00:00 2001 From: clonylu Date: Tue, 15 Sep 2026 16:11:55 +0800 Subject: [PATCH 2/7] test(router): cover passthrough stream_timeout without patching proxy globals --- .../test_pass_through_endpoints.py | 14 ++--- tests/test_litellm/test_router.py | 56 ++++++++++--------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index d697a114613..7f8663ea860 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1167,14 +1167,14 @@ def test_resolve_llm_passthrough_timeout_stream_timeout_precedence(): ) == 90.0 ) - with patch("litellm.proxy.proxy_server.general_settings", {}): - assert ( - resolve_llm_passthrough_timeout( - litellm_params={"stream_timeout": 1800}, - router_stream_timeout=1800, - ) - == DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS + assert ( + resolve_llm_passthrough_timeout( + litellm_params={"stream_timeout": 1800}, + router_timeout=120, + router_stream_timeout=1800, ) + == 120.0 + ) @pytest.mark.asyncio diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 70ce182c028..4c39f8ba4e4 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5494,6 +5494,7 @@ def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): "litellm_params": { "model": "anthropic/claude-sonnet-4-5", "api_key": "fake-key", + "timeout": 60, "stream_timeout": 1800, }, }, @@ -5505,37 +5506,42 @@ def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): }, }, ], + timeout=120, stream_timeout=900, ) per_deployment, router_default = router.model_list - with patch( - "litellm.proxy.proxy_server.general_settings", - {"pass_through_request_timeout": 6}, - ): - kwargs: dict = {"stream": True} - router._update_kwargs_with_deployment( - deployment=per_deployment, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 1800.0 + kwargs: dict = {"stream": True} + router._update_kwargs_with_deployment( + deployment=per_deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 1800.0 - kwargs = {"stream": True} - router._update_kwargs_with_deployment( - deployment=router_default, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 900.0 + kwargs = {"stream": True} + router._update_kwargs_with_deployment( + deployment=router_default, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 900.0 - kwargs = {"stream": False} - router._update_kwargs_with_deployment( - deployment=per_deployment, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 6.0 + kwargs = {"stream": False} + router._update_kwargs_with_deployment( + deployment=per_deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 60.0 + + kwargs = {"stream": False} + router._update_kwargs_with_deployment( + deployment=router_default, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 120.0 @pytest.mark.asyncio From 73fddb999e3849ffa31897426a82353ac3705521 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 18 Sep 2026 11:53:26 -0700 Subject: [PATCH 3/7] fix(router): resolve stream_timeout before generic timeouts on the passthrough route --- litellm/passthrough/timeout_utils.py | 56 +++++++++-------- litellm/router.py | 4 +- .../test_pass_through_endpoints.py | 38 ++++++------ tests/test_litellm/test_router.py | 61 +++++++++---------- 4 files changed, 80 insertions(+), 79 deletions(-) diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index fb649a9eeaf..0170d93c156 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -1,9 +1,20 @@ import sys from typing import Final +from pydantic import BaseModel, ConfigDict + DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0 +class _TimeoutFields(BaseModel): + model_config = ConfigDict(frozen=True) + + stream: bool = False + stream_timeout: float | None = None + timeout: float | None = None + request_timeout: float | None = None + + def resolve_pass_through_request_timeout( endpoint_timeout: float | None = None, ) -> float: @@ -33,8 +44,8 @@ def resolve_pass_through_request_timeout( def resolve_llm_passthrough_timeout( kwargs: dict | None = None, litellm_params: dict | None = None, - router_timeout: float | None = None, - router_stream_timeout: float | None = None, + router_timeout: float | str | None = None, + router_stream_timeout: float | str | None = None, ) -> float: """ Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse, @@ -44,28 +55,23 @@ def resolve_llm_passthrough_timeout( timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout -> 600s default. - Streaming (``kwargs["stream"]`` truthy) additionally consults ``stream_timeout`` at each - level before the non-streaming key, matching ``Router._get_stream_timeout`` on the - completion route: kwargs stream_timeout -> kwargs timeout/request_timeout -> - litellm_params stream_timeout -> litellm_params timeout/request_timeout -> - router_stream_timeout -> router_timeout -> pass_through_request_timeout -> 600s. + Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before + any generic timeout, matching ``Router._get_stream_timeout`` on the completion route: + kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the + non-streaming chain above. """ - kwargs = kwargs or {} - litellm_params = litellm_params or {} - is_stream: Final[bool] = bool(kwargs.get("stream", False)) - - keys: Final[tuple[str, ...]] = ( - ("stream_timeout", "timeout", "request_timeout") if is_stream else ("timeout", "request_timeout") + request: Final = _TimeoutFields.model_validate(kwargs or {}) + deployment: Final = _TimeoutFields.model_validate(litellm_params or {}) + stream_candidates: Final = ( + (request.stream_timeout, deployment.stream_timeout, router_stream_timeout) if request.stream else () ) - for source in (kwargs, litellm_params): - for key in keys: - val = source.get(key) - if val is not None: - return float(val) - - if is_stream and router_stream_timeout is not None: - return float(router_stream_timeout) - if router_timeout is not None: - return float(router_timeout) - - return resolve_pass_through_request_timeout() + candidates: Final = ( + *stream_candidates, + request.timeout, + request.request_timeout, + deployment.timeout, + deployment.request_timeout, + router_timeout, + ) + resolved: Final = next((float(val) for val in candidates if val is not None), None) + return resolved if resolved is not None else resolve_pass_through_request_timeout() diff --git a/litellm/router.py b/litellm/router.py index 7ce7ba30502..a5523d7af79 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3880,7 +3880,9 @@ class Router: float(self._explicit_timeout) if isinstance(self._explicit_timeout, (int, float)) else None ) _router_stream_timeout: Final = ( - float(self.stream_timeout) if isinstance(self.stream_timeout, (int, float)) else None + self.stream_timeout + if self.stream_timeout is not None + else self.default_litellm_params.get("stream_timeout") ) kwargs["timeout"] = resolve_llm_passthrough_timeout( kwargs=kwargs, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 7f8663ea860..54336800db4 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1120,7 +1120,6 @@ def test_resolve_llm_passthrough_timeout_precedence(): def test_resolve_llm_passthrough_timeout_stream_timeout_precedence(): - # streaming: stream_timeout wins at each level, then falls through to the non-stream keys assert ( resolve_llm_passthrough_timeout( kwargs={"stream": True, "stream_timeout": 1800, "timeout": 45}, @@ -1129,36 +1128,35 @@ def test_resolve_llm_passthrough_timeout_stream_timeout_precedence(): ) assert ( resolve_llm_passthrough_timeout( - kwargs={"stream": True}, + kwargs={"stream": True, "timeout": 45}, litellm_params={"stream_timeout": 1800, "timeout": 90}, ) == 1800.0 ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True, "timeout": 45}, + litellm_params={"timeout": 90}, + router_timeout=120, + router_stream_timeout=1800, + ) + == 1800.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + router_stream_timeout="1800", + ) + == 1800.0 + ) assert ( resolve_llm_passthrough_timeout( kwargs={"stream": True}, litellm_params={"timeout": 90}, - router_stream_timeout=1800, + router_timeout=120, ) == 90.0 ) - assert ( - resolve_llm_passthrough_timeout( - kwargs={"stream": True}, - router_timeout=120, - router_stream_timeout=1800, - ) - == 1800.0 - ) - assert ( - resolve_llm_passthrough_timeout( - kwargs={"stream": True}, - router_timeout=120, - ) - == 120.0 - ) - - # non-streaming: stream_timeout is ignored everywhere assert ( resolve_llm_passthrough_timeout( kwargs={"stream": False, "stream_timeout": 1800}, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 4c39f8ba4e4..e92e5de23dc 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5480,13 +5480,17 @@ def test_update_kwargs_with_deployment_uses_pass_through_request_timeout(): assert kwargs["timeout"] == 6.0 +def _passthrough_timeout(router: litellm.Router, deployment: dict, stream: bool) -> float: + kwargs: Final[dict] = {"stream": stream} + router._update_kwargs_with_deployment( + deployment=deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + return kwargs["timeout"] + + def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): - """ - The SDK-native passthrough route (anthropic /v1/messages, bedrock /converse) resolves - its upstream timeout separately from the completion route. A streaming call must get - stream_timeout (deployment litellm_params first, then router_settings), while a - non-streaming call on the same deployment keeps the non-stream resolution. - """ router = litellm.Router( model_list=[ { @@ -5503,6 +5507,7 @@ def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): "litellm_params": { "model": "anthropic/claude-sonnet-4-5", "api_key": "fake-key", + "timeout": 60, }, }, ], @@ -5511,37 +5516,27 @@ def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): ) per_deployment, router_default = router.model_list - kwargs: dict = {"stream": True} - router._update_kwargs_with_deployment( - deployment=per_deployment, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 1800.0 + assert _passthrough_timeout(router, per_deployment, stream=True) == 1800.0 + assert _passthrough_timeout(router, router_default, stream=True) == 900.0 + assert _passthrough_timeout(router, per_deployment, stream=False) == 60.0 + assert _passthrough_timeout(router, router_default, stream=False) == 60.0 - kwargs = {"stream": True} - router._update_kwargs_with_deployment( - deployment=router_default, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 900.0 - kwargs = {"stream": False} - router._update_kwargs_with_deployment( - deployment=per_deployment, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", +def test_update_kwargs_with_deployment_passthrough_router_stream_timeout_sources(): + deployment: Final[dict] = { + "model_name": "anthropic-router-default", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "fake-key"}, + } + string_router = litellm.Router(model_list=[deployment], timeout=120, stream_timeout="900") + default_router = litellm.Router( + model_list=[deployment], + timeout=120, + default_litellm_params={"stream_timeout": 700}, ) - assert kwargs["timeout"] == 60.0 - kwargs = {"stream": False} - router._update_kwargs_with_deployment( - deployment=router_default, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 120.0 + assert _passthrough_timeout(string_router, string_router.model_list[0], stream=True) == 900.0 + assert _passthrough_timeout(default_router, default_router.model_list[0], stream=True) == 700.0 + assert _passthrough_timeout(default_router, default_router.model_list[0], stream=False) == 120.0 @pytest.mark.asyncio From d4ee62eb8c3083a48fb21f72da49c14a578dc993 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:29:32 -0700 Subject: [PATCH 4/7] refactor(passthrough): type the timeout resolver mapping parameters --- litellm/passthrough/timeout_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index 0170d93c156..829277105e3 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -1,4 +1,5 @@ import sys +from collections.abc import Mapping from typing import Final from pydantic import BaseModel, ConfigDict @@ -42,8 +43,8 @@ def resolve_pass_through_request_timeout( def resolve_llm_passthrough_timeout( - kwargs: dict | None = None, - litellm_params: dict | None = None, + kwargs: Mapping[str, object] | None = None, + litellm_params: Mapping[str, object] | None = None, router_timeout: float | str | None = None, router_stream_timeout: float | str | None = None, ) -> float: From 58beea227547cccf54dd70eb1f7ae3024b8a1a03 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 18 Sep 2026 12:45:54 -0700 Subject: [PATCH 5/7] fix(passthrough): read the stream flag by truthiness in the timeout resolver --- litellm/passthrough/timeout_utils.py | 4 ++-- .../test_pass_through_endpoints.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index 829277105e3..9284f7143b0 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -10,7 +10,6 @@ DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0 class _TimeoutFields(BaseModel): model_config = ConfigDict(frozen=True) - stream: bool = False stream_timeout: float | None = None timeout: float | None = None request_timeout: float | None = None @@ -61,10 +60,11 @@ def resolve_llm_passthrough_timeout( kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above. """ + streaming: Final = bool((kwargs or {}).get("stream")) request: Final = _TimeoutFields.model_validate(kwargs or {}) deployment: Final = _TimeoutFields.model_validate(litellm_params or {}) stream_candidates: Final = ( - (request.stream_timeout, deployment.stream_timeout, router_stream_timeout) if request.stream else () + (request.stream_timeout, deployment.stream_timeout, router_stream_timeout) if streaming else () ) candidates: Final = ( *stream_candidates, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 54336800db4..abff897c4f5 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1175,6 +1175,20 @@ def test_resolve_llm_passthrough_timeout_stream_timeout_precedence(): ) +@pytest.mark.parametrize( + "stream, expected", + [(None, 90.0), (0, 90.0), ("", 90.0), (1, 1800.0), ("yes", 1800.0)], +) +def test_resolve_llm_passthrough_timeout_reads_stream_by_truthiness(stream: object, expected: float): + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": stream}, + litellm_params={"stream_timeout": 1800, "timeout": 90}, + ) + == expected + ) + + @pytest.mark.asyncio async def test_pass_through_request_uses_resolved_timeout(): with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: From 74e9fb23233c8d3bfeed73485bc0270da79370d3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 18 Sep 2026 13:07:33 -0700 Subject: [PATCH 6/7] fix(passthrough): validate only the winning timeout value in the resolver --- litellm/passthrough/timeout_utils.py | 36 +++++++++---------- .../test_pass_through_endpoints.py | 20 +++++++++++ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index 9284f7143b0..fc67aa8c553 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -1,18 +1,14 @@ import sys from collections.abc import Mapping +from types import MappingProxyType from typing import Final -from pydantic import BaseModel, ConfigDict +from pydantic import TypeAdapter DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0 - -class _TimeoutFields(BaseModel): - model_config = ConfigDict(frozen=True) - - stream_timeout: float | None = None - timeout: float | None = None - request_timeout: float | None = None +_SECONDS: Final = TypeAdapter(float) +_NO_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) def resolve_pass_through_request_timeout( @@ -59,20 +55,24 @@ def resolve_llm_passthrough_timeout( any generic timeout, matching ``Router._get_stream_timeout`` on the completion route: kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above. + + Only the first set value is validated as seconds, so a value in a lower-precedence + field never fails the call. """ - streaming: Final = bool((kwargs or {}).get("stream")) - request: Final = _TimeoutFields.model_validate(kwargs or {}) - deployment: Final = _TimeoutFields.model_validate(litellm_params or {}) + request: Final = kwargs if kwargs is not None else _NO_PARAMS + deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS stream_candidates: Final = ( - (request.stream_timeout, deployment.stream_timeout, router_stream_timeout) if streaming else () + (request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout) + if request.get("stream") + else () ) candidates: Final = ( *stream_candidates, - request.timeout, - request.request_timeout, - deployment.timeout, - deployment.request_timeout, + request.get("timeout"), + request.get("request_timeout"), + deployment.get("timeout"), + deployment.get("request_timeout"), router_timeout, ) - resolved: Final = next((float(val) for val in candidates if val is not None), None) - return resolved if resolved is not None else resolve_pass_through_request_timeout() + winner: Final = next((val for val in candidates if val is not None), None) + return resolve_pass_through_request_timeout() if winner is None else _SECONDS.validate_python(winner) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index abff897c4f5..13cd41278b9 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest from fastapi import Request, Response, UploadFile +from pydantic import ValidationError from starlette.datastructures import FormData, Headers, QueryParams from starlette.datastructures import UploadFile as StarletteUploadFile @@ -1189,6 +1190,25 @@ def test_resolve_llm_passthrough_timeout_reads_stream_by_truthiness(stream: obje ) +@pytest.mark.parametrize( + "kwargs, litellm_params, expected", + [ + ({"stream": True, "stream_timeout": 1800, "timeout": httpx.Timeout(30.0)}, {}, 1800.0), + ({"stream": False}, {"stream_timeout": httpx.Timeout(30.0), "timeout": 90}, 90.0), + ({"timeout": 45}, {"request_timeout": httpx.Timeout(30.0)}, 45.0), + ], +) +def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value( + kwargs: dict[str, object], litellm_params: dict[str, object], expected: float +): + assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected + + +def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner(): + with pytest.raises(ValidationError): + resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)}) + + @pytest.mark.asyncio async def test_pass_through_request_uses_resolved_timeout(): with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: From e75ad61fceaf08b49830dc4c737db2ea7a28f49c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 18 Sep 2026 13:32:43 -0700 Subject: [PATCH 7/7] fix(router): rank litellm_settings.request_timeout on the passthrough route like the completion route --- litellm/router.py | 8 +++++++- tests/test_litellm/test_router.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index a5523d7af79..e3e9454ddce 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3877,11 +3877,17 @@ class Router: ) _router_timeout: Final = ( - float(self._explicit_timeout) if isinstance(self._explicit_timeout, (int, float)) else None + self.request_timeout + if self.request_timeout is not None + else float(self._explicit_timeout) + if isinstance(self._explicit_timeout, (int, float)) + else None ) _router_stream_timeout: Final = ( self.stream_timeout if self.stream_timeout is not None + else self.request_timeout + if self.request_timeout is not None else self.default_litellm_params.get("stream_timeout") ) kwargs["timeout"] = resolve_llm_passthrough_timeout( diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index e92e5de23dc..148ca0b0ffa 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8230,6 +8230,16 @@ class TestRouterRequestTimeoutPropagation: == 60 ) + def test_passthrough_prefers_request_timeout_over_router_timeout(self, explicit_request_timeout): + router = self._make_router(timeout=330) + deployment: Final = router.model_list[0] + assert _passthrough_timeout(router, deployment, stream=False) == 300.0 + assert _passthrough_timeout(router, deployment, stream=True) == 300.0 + + def test_passthrough_stream_timeout_still_wins_over_request_timeout(self, explicit_request_timeout): + router = self._make_router(timeout=330, stream_timeout=45) + assert _passthrough_timeout(router, router.model_list[0], stream=True) == 45.0 + # --------------------------------------------------------------------------- # Deferred-stream eager-fetch tests