From 2019f39d709073095b4c220497bbf712179d63ab Mon Sep 17 00:00:00 2001 From: shubh3ai Date: Tue, 31 Mar 2026 14:49:59 -0700 Subject: [PATCH] feat: add timeout to passthrough route --- litellm/passthrough/main.py | 17 +- .../passthrough/test_passthrough_main.py | 153 ++++++++++++++++++ 2 files changed, 169 insertions(+), 1 deletion(-) diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index edee50bdfc4..d3f92653801 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -25,7 +25,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.passthrough.utils import CommonUtils -from litellm.utils import client +from litellm.utils import client, supports_httpx_timeout base_llm_http_handler = BaseLLMHTTPHandler() from .utils import BasePassthroughUtils @@ -52,6 +52,7 @@ async def allm_passthrough_route( json: Optional[Any] = None, params: Optional[QueryParamTypes] = None, cookies: Optional[CookieTypes] = None, + timeout: Optional[Union[float, int, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, **kwargs, ) -> Union[httpx.Response, AsyncGenerator[Any, Any]]: @@ -98,6 +99,7 @@ async def allm_passthrough_route( json=json, params=params, cookies=cookies, + timeout=timeout, client=client, **kwargs, ) @@ -178,6 +180,7 @@ def llm_passthrough_route( json: Optional[Any] = None, params: Optional[QueryParamTypes] = None, cookies: Optional[CookieTypes] = None, + timeout: Optional[Union[float, int, httpx.Timeout]] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, **kwargs, ) -> Union[ @@ -286,6 +289,17 @@ def llm_passthrough_route( if json and isinstance(json, dict) and "model" in json: json["model"] = model + ### TIMEOUT LOGIC ### + timeout = timeout or kwargs.get("request_timeout", 600) or 600 + if ( + custom_llm_provider is not None + and isinstance(timeout, httpx.Timeout) + and not supports_httpx_timeout(custom_llm_provider) + ): + timeout = timeout.read or 600 + elif not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) + request = client.client.build_request( method=method, url=updated_url, @@ -296,6 +310,7 @@ def llm_passthrough_route( params=params, headers=headers, cookies=cookies, + timeout=timeout, ) ## IS STREAMING REQUEST diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index c84d32d48f2..715bbabe0f4 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -510,6 +510,159 @@ def test_azure_with_custom_api_base_and_key(): assert response.status_code == 200 # type: ignore[union-attr] +def test_timeout_param_forwarded_to_build_request(): + """ + Verify that the `timeout` parameter is resolved and forwarded to + build_request so httpx enforces it on the underlying HTTP call. + The @client decorator reads kwargs["timeout"] on exception and attaches + it to the exception for the router's fallback logic. + """ + client = HTTPHandler() + + mock_provider_config = MagicMock() + mock_provider_config.get_complete_url.return_value = ( + httpx.URL("https://api.example.com/v1/chat/completions"), + "https://api.example.com", + ) + mock_provider_config.get_api_key.return_value = "test-key" + mock_provider_config.validate_environment.return_value = {} + mock_provider_config.sign_request.return_value = ({}, None) + mock_provider_config.is_streaming_request.return_value = False + + with patch( + "litellm.utils.ProviderConfigManager.get_provider_passthrough_config", + return_value=mock_provider_config, + ), patch( + "litellm.litellm_core_utils.get_litellm_params.get_litellm_params", + return_value={}, + ), patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("my-model", "openai", "test-key", "https://api.example.com"), + ), patch.object( + client.client, "send", return_value=MagicMock(status_code=200) + ), patch.object( + client.client, "build_request" + ) as mock_build_request: + mock_logging_obj = MagicMock() + mock_logging_obj.update_environment_variables = MagicMock() + + llm_passthrough_route( + model="openai/my-model", + endpoint="v1/chat/completions", + method="POST", + custom_llm_provider="openai", + json={"model": "my-model", "messages": [{"role": "user", "content": "Hi"}]}, + timeout=30, + client=client, + litellm_logging_obj=mock_logging_obj, + ) + + mock_build_request.assert_called_once() + call_kwargs = mock_build_request.call_args.kwargs + assert call_kwargs["timeout"] == 30.0 + + +def test_timeout_defaults_to_600_when_not_provided(): + """ + When no timeout is passed, the passthrough should default to 600s + (matching the completion endpoint behaviour). + """ + client = HTTPHandler() + + mock_provider_config = MagicMock() + mock_provider_config.get_complete_url.return_value = ( + httpx.URL("https://api.example.com/v1/chat/completions"), + "https://api.example.com", + ) + mock_provider_config.get_api_key.return_value = "test-key" + mock_provider_config.validate_environment.return_value = {} + mock_provider_config.sign_request.return_value = ({}, None) + mock_provider_config.is_streaming_request.return_value = False + + with patch( + "litellm.utils.ProviderConfigManager.get_provider_passthrough_config", + return_value=mock_provider_config, + ), patch( + "litellm.litellm_core_utils.get_litellm_params.get_litellm_params", + return_value={}, + ), patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("my-model", "openai", "test-key", "https://api.example.com"), + ), patch.object( + client.client, "send", return_value=MagicMock(status_code=200) + ), patch.object( + client.client, "build_request" + ) as mock_build_request: + mock_logging_obj = MagicMock() + mock_logging_obj.update_environment_variables = MagicMock() + + llm_passthrough_route( + model="openai/my-model", + endpoint="v1/chat/completions", + method="POST", + custom_llm_provider="openai", + json={"model": "my-model", "messages": [{"role": "user", "content": "Hi"}]}, + client=client, + litellm_logging_obj=mock_logging_obj, + ) + + mock_build_request.assert_called_once() + call_kwargs = mock_build_request.call_args.kwargs + assert call_kwargs["timeout"] == 600.0 + + +def test_timeout_httpx_timeout_object_forwarded(): + """ + When an httpx.Timeout object is passed, it should be forwarded + directly to build_request without conversion. + """ + client = HTTPHandler() + + mock_provider_config = MagicMock() + mock_provider_config.get_complete_url.return_value = ( + httpx.URL("https://api.example.com/v1/chat/completions"), + "https://api.example.com", + ) + mock_provider_config.get_api_key.return_value = "test-key" + mock_provider_config.validate_environment.return_value = {} + mock_provider_config.sign_request.return_value = ({}, None) + mock_provider_config.is_streaming_request.return_value = False + + custom_timeout = httpx.Timeout(timeout=45.0, connect=10.0) + + with patch( + "litellm.utils.ProviderConfigManager.get_provider_passthrough_config", + return_value=mock_provider_config, + ), patch( + "litellm.litellm_core_utils.get_litellm_params.get_litellm_params", + return_value={}, + ), patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("my-model", "openai", "test-key", "https://api.example.com"), + ), patch.object( + client.client, "send", return_value=MagicMock(status_code=200) + ), patch.object( + client.client, "build_request" + ) as mock_build_request: + mock_logging_obj = MagicMock() + mock_logging_obj.update_environment_variables = MagicMock() + + llm_passthrough_route( + model="openai/my-model", + endpoint="v1/chat/completions", + method="POST", + custom_llm_provider="openai", + json={"model": "my-model", "messages": [{"role": "user", "content": "Hi"}]}, + timeout=custom_timeout, + client=client, + litellm_logging_obj=mock_logging_obj, + ) + + mock_build_request.assert_called_once() + call_kwargs = mock_build_request.call_args.kwargs + assert call_kwargs["timeout"] is custom_timeout + + def test_content_param_forwarded_to_build_request(): """ Regression test: the `content` parameter passed to llm_passthrough_route