Merge pull request #41875 from BerriAI/litellm_passthrough_stream_timeout

fix(router): honor stream_timeout on the SDK-native passthrough route (/v1/messages, /converse)

litellm/passthrough/timeout_utils.py

@@ -1,8 +1,15 @@
 import sys
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Final
 
+from pydantic import TypeAdapter
+
 DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0
 
+_SECONDS: Final = TypeAdapter(float)
+_NO_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
+
 
 def resolve_pass_through_request_timeout(
     endpoint_timeout: float | None = None,
@@ -31,26 +38,41 @@ def resolve_pass_through_request_timeout(
 
 
 def resolve_llm_passthrough_timeout(
-    kwargs: dict | None = None,
-    litellm_params: dict | None = None,
-    router_timeout: float | None = None,
+    kwargs: Mapping[str, object] | None = None,
+    litellm_params: Mapping[str, object] | None = None,
+    router_timeout: float | str | None = None,
+    router_stream_timeout: float | str | None = None,
 ) -> float:
     """
-    Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse).
+    Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse,
+    Anthropic /v1/messages).
 
-    Precedence: kwargs timeout/request_timeout -> litellm_params timeout/request_timeout
-    -> router_timeout -> general_settings.pass_through_request_timeout -> 600s default.
+    Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params
+    timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout
+    -> 600s default.
+
+    Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before
+    any generic timeout, matching ``Router._get_stream_timeout`` on the completion route:
+    kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the
+    non-streaming chain above.
+
+    Only the first set value is validated as seconds, so a value in a lower-precedence
+    field never fails the call.
     """
-    kwargs = kwargs or {}
-    litellm_params = litellm_params or {}
-
-    for source in (kwargs, litellm_params):
-        for key in ("timeout", "request_timeout"):
-            val = source.get(key)
-            if val is not None:
-                return float(val)
-
-    if router_timeout is not None:
-        return float(router_timeout)
-
-    return resolve_pass_through_request_timeout()
+    request: Final = kwargs if kwargs is not None else _NO_PARAMS
+    deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS
+    stream_candidates: Final = (
+        (request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout)
+        if request.get("stream")
+        else ()
+    )
+    candidates: Final = (
+        *stream_candidates,
+        request.get("timeout"),
+        request.get("request_timeout"),
+        deployment.get("timeout"),
+        deployment.get("request_timeout"),
+        router_timeout,
+    )
+    winner: Final = next((val for val in candidates if val is not None), None)
+    return resolve_pass_through_request_timeout() if winner is None else _SECONDS.validate_python(winner)

litellm/router.py

@@ -3940,12 +3940,24 @@ class Router:
             )
 
             _router_timeout: Final = (
-                float(self._explicit_timeout) if isinstance(self._explicit_timeout, (int, float)) else None
+                self.request_timeout
+                if self.request_timeout is not None
+                else float(self._explicit_timeout)
+                if isinstance(self._explicit_timeout, (int, float))
+                else None
+            )
+            _router_stream_timeout: Final = (
+                self.stream_timeout
+                if self.stream_timeout is not None
+                else self.request_timeout
+                if self.request_timeout is not None
+                else self.default_litellm_params.get("stream_timeout")
             )
             kwargs["timeout"] = resolve_llm_passthrough_timeout(
                 kwargs=kwargs,
                 litellm_params=deployment["litellm_params"],
                 router_timeout=_router_timeout,
+                router_stream_timeout=_router_stream_timeout,
             )
         else:
             kwargs["timeout"] = self._get_timeout(kwargs=kwargs, data=deployment["litellm_params"])

tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py

@@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import httpx
 import pytest
 from fastapi import Request, Response, UploadFile
+from pydantic import ValidationError
 from starlette.datastructures import FormData, Headers, QueryParams
 from starlette.datastructures import UploadFile as StarletteUploadFile
 
@@ -1155,6 +1156,95 @@ def test_resolve_llm_passthrough_timeout_precedence():
         assert resolve_llm_passthrough_timeout() == 6.0
 
 
+def test_resolve_llm_passthrough_timeout_stream_timeout_precedence():
+    assert (
+        resolve_llm_passthrough_timeout(
+            kwargs={"stream": True, "stream_timeout": 1800, "timeout": 45},
+        )
+        == 1800.0
+    )
+    assert (
+        resolve_llm_passthrough_timeout(
+            kwargs={"stream": True, "timeout": 45},
+            litellm_params={"stream_timeout": 1800, "timeout": 90},
+        )
+        == 1800.0
+    )
+    assert (
+        resolve_llm_passthrough_timeout(
+            kwargs={"stream": True, "timeout": 45},
+            litellm_params={"timeout": 90},
+            router_timeout=120,
+            router_stream_timeout=1800,
+        )
+        == 1800.0
+    )
+    assert (
+        resolve_llm_passthrough_timeout(
+            kwargs={"stream": True},
+            router_stream_timeout="1800",
+        )
+        == 1800.0
+    )
+    assert (
+        resolve_llm_passthrough_timeout(
+            kwargs={"stream": True},
+            litellm_params={"timeout": 90},
+            router_timeout=120,
+        )
+        == 90.0
+    )
+    assert (
+        resolve_llm_passthrough_timeout(
+            kwargs={"stream": False, "stream_timeout": 1800},
+            litellm_params={"stream_timeout": 1800, "timeout": 90},
+            router_stream_timeout=1800,
+        )
+        == 90.0
+    )
+    assert (
+        resolve_llm_passthrough_timeout(
+            litellm_params={"stream_timeout": 1800},
+            router_timeout=120,
+            router_stream_timeout=1800,
+        )
+        == 120.0
+    )
+
+
+@pytest.mark.parametrize(
+    "stream, expected",
+    [(None, 90.0), (0, 90.0), ("", 90.0), (1, 1800.0), ("yes", 1800.0)],
+)
+def test_resolve_llm_passthrough_timeout_reads_stream_by_truthiness(stream: object, expected: float):
+    assert (
+        resolve_llm_passthrough_timeout(
+            kwargs={"stream": stream},
+            litellm_params={"stream_timeout": 1800, "timeout": 90},
+        )
+        == expected
+    )
+
+
+@pytest.mark.parametrize(
+    "kwargs, litellm_params, expected",
+    [
+        ({"stream": True, "stream_timeout": 1800, "timeout": httpx.Timeout(30.0)}, {}, 1800.0),
+        ({"stream": False}, {"stream_timeout": httpx.Timeout(30.0), "timeout": 90}, 90.0),
+        ({"timeout": 45}, {"request_timeout": httpx.Timeout(30.0)}, 45.0),
+    ],
+)
+def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value(
+    kwargs: dict[str, object], litellm_params: dict[str, object], expected: float
+):
+    assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected
+
+
+def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner():
+    with pytest.raises(ValidationError):
+        resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)})
+
+
 @pytest.mark.asyncio
 async def test_pass_through_request_uses_resolved_timeout():
     with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging:

tests/test_litellm/test_router.py

@@ -5538,6 +5538,65 @@ def test_update_kwargs_with_deployment_uses_pass_through_request_timeout():
     assert kwargs["timeout"] == 6.0
 
 
+def _passthrough_timeout(router: litellm.Router, deployment: dict, stream: bool) -> float:
+    kwargs: Final[dict] = {"stream": stream}
+    router._update_kwargs_with_deployment(
+        deployment=deployment,
+        kwargs=kwargs,
+        function_name="_ageneric_api_call_with_fallbacks",
+    )
+    return kwargs["timeout"]
+
+
+def test_update_kwargs_with_deployment_passthrough_honors_stream_timeout():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "anthropic-with-stream-timeout",
+                "litellm_params": {
+                    "model": "anthropic/claude-sonnet-4-5",
+                    "api_key": "fake-key",
+                    "timeout": 60,
+                    "stream_timeout": 1800,
+                },
+            },
+            {
+                "model_name": "anthropic-router-default",
+                "litellm_params": {
+                    "model": "anthropic/claude-sonnet-4-5",
+                    "api_key": "fake-key",
+                    "timeout": 60,
+                },
+            },
+        ],
+        timeout=120,
+        stream_timeout=900,
+    )
+    per_deployment, router_default = router.model_list
+
+    assert _passthrough_timeout(router, per_deployment, stream=True) == 1800.0
+    assert _passthrough_timeout(router, router_default, stream=True) == 900.0
+    assert _passthrough_timeout(router, per_deployment, stream=False) == 60.0
+    assert _passthrough_timeout(router, router_default, stream=False) == 60.0
+
+
+def test_update_kwargs_with_deployment_passthrough_router_stream_timeout_sources():
+    deployment: Final[dict] = {
+        "model_name": "anthropic-router-default",
+        "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "fake-key"},
+    }
+    string_router = litellm.Router(model_list=[deployment], timeout=120, stream_timeout="900")
+    default_router = litellm.Router(
+        model_list=[deployment],
+        timeout=120,
+        default_litellm_params={"stream_timeout": 700},
+    )
+
+    assert _passthrough_timeout(string_router, string_router.model_list[0], stream=True) == 900.0
+    assert _passthrough_timeout(default_router, default_router.model_list[0], stream=True) == 700.0
+    assert _passthrough_timeout(default_router, default_router.model_list[0], stream=False) == 120.0
+
+
 @pytest.mark.asyncio
 async def test_router_acompletion_with_unknown_model_and_default_fallback():
     """
@@ -8229,6 +8288,16 @@ class TestRouterRequestTimeoutPropagation:
             == 60
         )
 
+    def test_passthrough_prefers_request_timeout_over_router_timeout(self, explicit_request_timeout):
+        router = self._make_router(timeout=330)
+        deployment: Final = router.model_list[0]
+        assert _passthrough_timeout(router, deployment, stream=False) == 300.0
+        assert _passthrough_timeout(router, deployment, stream=True) == 300.0
+
+    def test_passthrough_stream_timeout_still_wins_over_request_timeout(self, explicit_request_timeout):
+        router = self._make_router(timeout=330, stream_timeout=45)
+        assert _passthrough_timeout(router, router.model_list[0], stream=True) == 45.0
+
 
 # ---------------------------------------------------------------------------
 # Deferred-stream eager-fetch tests