diff --git a/litellm/passthrough/timeout_utils.py b/litellm/passthrough/timeout_utils.py index 39127d19183..fb649a9eeaf 100644 --- a/litellm/passthrough/timeout_utils.py +++ b/litellm/passthrough/timeout_utils.py @@ -34,22 +34,37 @@ def resolve_llm_passthrough_timeout( kwargs: dict | None = None, litellm_params: dict | None = None, router_timeout: float | None = None, + router_stream_timeout: float | None = None, ) -> float: """ - Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse). + Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse, + Anthropic /v1/messages). - Precedence: kwargs timeout/request_timeout -> litellm_params timeout/request_timeout - -> router_timeout -> general_settings.pass_through_request_timeout -> 600s default. + Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params + timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout + -> 600s default. + + Streaming (``kwargs["stream"]`` truthy) additionally consults ``stream_timeout`` at each + level before the non-streaming key, matching ``Router._get_stream_timeout`` on the + completion route: kwargs stream_timeout -> kwargs timeout/request_timeout -> + litellm_params stream_timeout -> litellm_params timeout/request_timeout -> + router_stream_timeout -> router_timeout -> pass_through_request_timeout -> 600s. """ kwargs = kwargs or {} litellm_params = litellm_params or {} + is_stream: Final[bool] = bool(kwargs.get("stream", False)) + keys: Final[tuple[str, ...]] = ( + ("stream_timeout", "timeout", "request_timeout") if is_stream else ("timeout", "request_timeout") + ) for source in (kwargs, litellm_params): - for key in ("timeout", "request_timeout"): + for key in keys: val = source.get(key) if val is not None: return float(val) + if is_stream and router_stream_timeout is not None: + return float(router_stream_timeout) if router_timeout is not None: return float(router_timeout) diff --git a/litellm/router.py b/litellm/router.py index 1665583386f..7ce7ba30502 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3879,10 +3879,14 @@ class Router: _router_timeout: Final = ( float(self._explicit_timeout) if isinstance(self._explicit_timeout, (int, float)) else None ) + _router_stream_timeout: Final = ( + float(self.stream_timeout) if isinstance(self.stream_timeout, (int, float)) else None + ) kwargs["timeout"] = resolve_llm_passthrough_timeout( kwargs=kwargs, litellm_params=deployment["litellm_params"], router_timeout=_router_timeout, + router_stream_timeout=_router_stream_timeout, ) else: kwargs["timeout"] = self._get_timeout(kwargs=kwargs, data=deployment["litellm_params"]) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index d57bed430c1..d697a114613 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1119,6 +1119,64 @@ def test_resolve_llm_passthrough_timeout_precedence(): assert resolve_llm_passthrough_timeout() == 6.0 +def test_resolve_llm_passthrough_timeout_stream_timeout_precedence(): + # streaming: stream_timeout wins at each level, then falls through to the non-stream keys + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True, "stream_timeout": 1800, "timeout": 45}, + ) + == 1800.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + litellm_params={"stream_timeout": 1800, "timeout": 90}, + ) + == 1800.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + litellm_params={"timeout": 90}, + router_stream_timeout=1800, + ) + == 90.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + router_timeout=120, + router_stream_timeout=1800, + ) + == 1800.0 + ) + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": True}, + router_timeout=120, + ) + == 120.0 + ) + + # non-streaming: stream_timeout is ignored everywhere + assert ( + resolve_llm_passthrough_timeout( + kwargs={"stream": False, "stream_timeout": 1800}, + litellm_params={"stream_timeout": 1800, "timeout": 90}, + router_stream_timeout=1800, + ) + == 90.0 + ) + with patch("litellm.proxy.proxy_server.general_settings", {}): + assert ( + resolve_llm_passthrough_timeout( + litellm_params={"stream_timeout": 1800}, + router_stream_timeout=1800, + ) + == DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS + ) + + @pytest.mark.asyncio async def test_pass_through_request_uses_resolved_timeout(): with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 3cabcd71627..70ce182c028 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5480,6 +5480,64 @@ def test_update_kwargs_with_deployment_uses_pass_through_request_timeout(): assert kwargs["timeout"] == 6.0 +def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout(): + """ + The SDK-native passthrough route (anthropic /v1/messages, bedrock /converse) resolves + its upstream timeout separately from the completion route. A streaming call must get + stream_timeout (deployment litellm_params first, then router_settings), while a + non-streaming call on the same deployment keeps the non-stream resolution. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "anthropic-with-stream-timeout", + "litellm_params": { + "model": "anthropic/claude-sonnet-4-5", + "api_key": "fake-key", + "stream_timeout": 1800, + }, + }, + { + "model_name": "anthropic-router-default", + "litellm_params": { + "model": "anthropic/claude-sonnet-4-5", + "api_key": "fake-key", + }, + }, + ], + stream_timeout=900, + ) + per_deployment, router_default = router.model_list + + with patch( + "litellm.proxy.proxy_server.general_settings", + {"pass_through_request_timeout": 6}, + ): + kwargs: dict = {"stream": True} + router._update_kwargs_with_deployment( + deployment=per_deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 1800.0 + + kwargs = {"stream": True} + router._update_kwargs_with_deployment( + deployment=router_default, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 900.0 + + kwargs = {"stream": False} + router._update_kwargs_with_deployment( + deployment=per_deployment, + kwargs=kwargs, + function_name="_ageneric_api_call_with_fallbacks", + ) + assert kwargs["timeout"] == 6.0 + + @pytest.mark.asyncio async def test_router_acompletion_with_unknown_model_and_default_fallback(): """