diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index fb649a9eeaf..0170d93c156 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -1,9 +1,20 @@ import sys from typing import Final +from pydantic import BaseModel, ConfigDict + DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0 +class _TimeoutFields(BaseModel): + model_config = ConfigDict(frozen=True) + + stream: bool = False + stream_timeout: float | None = None + timeout: float | None = None + request_timeout: float | None = None + + def resolve_pass_through_request_timeout( endpoint_timeout: float | None = None, ) -> float: @@ -33,8 +44,8 @@ def resolve_pass_through_request_timeout( def resolve_llm_passthrough_timeout( kwargs: dict | None = None, litellm_params: dict | None = None, - router_timeout: float | None = None, - router_stream_timeout: float | None = None, + router_timeout: float | str | None = None, + router_stream_timeout: float | str | None = None, ) -> float: """ Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse, @@ -44,28 +55,23 @@ def resolve_llm_passthrough_timeout( timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout -> 600s default. - Streaming (``kwargs["stream"]`` truthy) additionally consults ``stream_timeout`` at each - level before the non-streaming key, matching ``Router._get_stream_timeout`` on the - completion route: kwargs stream_timeout -> kwargs timeout/request_timeout -> - litellm_params stream_timeout -> litellm_params timeout/request_timeout -> - router_stream_timeout -> router_timeout -> pass_through_request_timeout -> 600s. + Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before + any generic timeout, matching ``Router._get_stream_timeout`` on the completion route: + kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the + non-streaming chain above. """ - kwargs = kwargs or {} - litellm_params = litellm_params or {} - is_stream: Final[bool] = bool(kwargs.get("stream", False)) - - keys: Final[tuple[str, ...]] = ( - ("stream_timeout", "timeout", "request_timeout") if is_stream else ("timeout", "request_timeout") + request: Final = _TimeoutFields.model_validate(kwargs or {}) + deployment: Final = _TimeoutFields.model_validate(litellm_params or {}) + stream_candidates: Final = ( + (request.stream_timeout, deployment.stream_timeout, router_stream_timeout) if request.stream else () ) - for source in (kwargs, litellm_params): - for key in keys: - val = source.get(key) - if val is not None: - return float(val) - - if is_stream and router_stream_timeout is not None: - return float(router_stream_timeout) - if router_timeout is not None: - return float(router_timeout) - - return resolve_pass_through_request_timeout() + candidates: Final = ( + *stream_candidates, + request.timeout, + request.request_timeout, + deployment.timeout, + deployment.request_timeout, + router_timeout, + ) + resolved: Final = next((float(val) for val in candidates if val is not None), None) + return resolved if resolved is not None else resolve_pass_through_request_timeout() diff --git a/litellm/router.py b/litellm/router.py index 7ce7ba30502..a5523d7af79 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3880,7 +3880,9 @@ class Router: float(self._explicit_timeout) if isinstance(self._explicit_timeout, (int, float)) else None ) _router_stream_timeout: Final = ( - float(self.stream_timeout) if isinstance(self.stream_timeout, (int, float)) else None + self.stream_timeout + if self.stream_timeout is not None + else self.default_litellm_params.get("stream_timeout") ) kwargs["timeout"] = resolve_llm_passthrough_timeout( kwargs=kwargs, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 7f8663ea860..54336800db4 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1120,7 +1120,6 @@ def test_resolve_llm_passthrough_timeout_precedence(): def test_resolve_llm_passthrough_timeout_stream_timeout_precedence(): - # streaming: stream_timeout wins at each level, then falls through to the non-stream keys assert ( resolve_llm_passthrough_timeout( kwargs={"stream": True, "stream_timeout": 1800, "timeout": 45}, @@ -1129,36 +1128,35 @@ def test_resolve_llm_passthrough_timeout_stream_timeout_precedence(): ) assert ( resolve_llm_passthrough_timeout( - kwargs={"stream": True}, + kwargs={"stream": True, "timeout": 45}, litellm_params={"stream_timeout": 1800, "timeout": 90}, ) == 1800.0 ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True, "timeout": 45}, + litellm_params={"timeout": 90}, + router_timeout=120, + router_stream_timeout=1800, + ) + == 1800.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + router_stream_timeout="1800", + ) + == 1800.0 + ) assert ( resolve_llm_passthrough_timeout( kwargs={"stream": True}, litellm_params={"timeout": 90}, - router_stream_timeout=1800, + router_timeout=120, ) == 90.0 ) - assert ( - resolve_llm_passthrough_timeout( - kwargs={"stream": True}, - router_timeout=120, - router_stream_timeout=1800, - ) - == 1800.0 - ) - assert ( - resolve_llm_passthrough_timeout( - kwargs={"stream": True}, - router_timeout=120, - ) - == 120.0 - ) - - # non-streaming: stream_timeout is ignored everywhere assert ( resolve_llm_passthrough_timeout( kwargs={"stream": False, "stream_timeout": 1800}, diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 4c39f8ba4e4..e92e5de23dc 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5480,13 +5480,17 @@ def test_update_kwargs_with_deployment_uses_pass_through_request_timeout(): assert kwargs["timeout"] == 6.0 +def _passthrough_timeout(router: litellm.Router, deployment: dict, stream: bool) -> float: + kwargs: Final[dict] = {"stream": stream} + router._update_kwargs_with_deployment( + deployment=deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + return kwargs["timeout"] + + def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): - """ - The SDK-native passthrough route (anthropic /v1/messages, bedrock /converse) resolves - its upstream timeout separately from the completion route. A streaming call must get - stream_timeout (deployment litellm_params first, then router_settings), while a - non-streaming call on the same deployment keeps the non-stream resolution. - """ router = litellm.Router( model_list=[ { @@ -5503,6 +5507,7 @@ def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): "litellm_params": { "model": "anthropic/claude-sonnet-4-5", "api_key": "fake-key", + "timeout": 60, }, }, ], @@ -5511,37 +5516,27 @@ def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): ) per_deployment, router_default = router.model_list - kwargs: dict = {"stream": True} - router._update_kwargs_with_deployment( - deployment=per_deployment, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 1800.0 + assert _passthrough_timeout(router, per_deployment, stream=True) == 1800.0 + assert _passthrough_timeout(router, router_default, stream=True) == 900.0 + assert _passthrough_timeout(router, per_deployment, stream=False) == 60.0 + assert _passthrough_timeout(router, router_default, stream=False) == 60.0 - kwargs = {"stream": True} - router._update_kwargs_with_deployment( - deployment=router_default, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 900.0 - kwargs = {"stream": False} - router._update_kwargs_with_deployment( - deployment=per_deployment, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", +def test_update_kwargs_with_deployment_passthrough_router_stream_timeout_sources(): + deployment: Final[dict] = { + "model_name": "anthropic-router-default", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "fake-key"}, + } + string_router = litellm.Router(model_list=[deployment], timeout=120, stream_timeout="900") + default_router = litellm.Router( + model_list=[deployment], + timeout=120, + default_litellm_params={"stream_timeout": 700}, ) - assert kwargs["timeout"] == 60.0 - kwargs = {"stream": False} - router._update_kwargs_with_deployment( - deployment=router_default, - kwargs=kwargs, - function_name="_ageneric_api_call_with_fallbacks", - ) - assert kwargs["timeout"] == 120.0 + assert _passthrough_timeout(string_router, string_router.model_list[0], stream=True) == 900.0 + assert _passthrough_timeout(default_router, default_router.model_list[0], stream=True) == 700.0 + assert _passthrough_timeout(default_router, default_router.model_list[0], stream=False) == 120.0 @pytest.mark.asyncio