diff --git a/litellm/main.py b/litellm/main.py index ddd37b47536..a8f70bcf8e7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1045,6 +1045,53 @@ def _build_custom_pricing_entry( return entry +def _resolve_completion_timeout( + timeout: Optional[Union[float, str, httpx.Timeout]], + kwargs: dict, + custom_llm_provider: str, +) -> Union[float, httpx.Timeout]: + """ + Resolve timeout inside completion(). + + Sources (first match wins): + + - **Model / deployment config:** the `timeout` argument (e.g. from router merging + per-model `litellm_params`, including a deployment-level `timeout`). + - **Model config alias:** ``kwargs["request_timeout"]`` when the caller passes the + per-model ``request_timeout`` field from model config (same idea as deployment + `litellm_params.request_timeout`). + - **Global proxy settings:** :attr:`litellm.request_timeout`, set from + ``litellm_settings.request_timeout`` when the proxy loads config. + - **Default:** ``600`` seconds if nothing above is set. + + Also accepts ``kwargs["timeout"]`` as a fallback when the named ``timeout`` argument + is omitted. + + If the resolved value is :class:`httpx.Timeout` and the provider does not support + passing it through (:func:`litellm.utils.supports_httpx_timeout`), coerce to a + float (read timeout, or ``600.0`` if read is unset). Otherwise numeric strings / + floats are coerced with ``float(...)``. + """ + if timeout is None: + timeout = kwargs.get("timeout") + if timeout is None: + timeout = kwargs.get("request_timeout") + if timeout is None: + timeout = getattr(litellm, "request_timeout", None) + if timeout is None: + timeout = 600 + if isinstance(timeout, httpx.Timeout) and not supports_httpx_timeout( + custom_llm_provider + ): + read_timeout = timeout.read + timeout = ( + float(read_timeout) if read_timeout is not None else 600.0 + ) # default 10 min timeout + elif not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + return timeout + + @tracer.wrap() @client def completion( # type: ignore # noqa: PLR0915 @@ -1400,14 +1447,11 @@ def completion( # type: ignore # noqa: PLR0915 ) # support region-based pricing for bedrock ### TIMEOUT LOGIC ### - timeout = timeout or kwargs.get("request_timeout", 600) or 600 - # set timeout for 10 minutes by default - if isinstance(timeout, httpx.Timeout) and not supports_httpx_timeout( - custom_llm_provider - ): - timeout = timeout.read or 600 # default 10 min timeout - elif not isinstance(timeout, httpx.Timeout): - timeout = float(timeout) # type: ignore + timeout = _resolve_completion_timeout( + timeout=timeout, + kwargs=kwargs, + custom_llm_provider=custom_llm_provider, + ) ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### if ( @@ -2667,6 +2711,7 @@ def completion( # type: ignore # noqa: PLR0915 provider_config=provider_config, ) else: + print("[debug] openai_chat_completions.completion", timeout) response = openai_chat_completions.completion( model=model, messages=messages, diff --git a/tests/local_testing/test_azure_anthropic_sync_post.py b/tests/local_testing/test_azure_anthropic_sync_post.py new file mode 100644 index 00000000000..14558d3bffb --- /dev/null +++ b/tests/local_testing/test_azure_anthropic_sync_post.py @@ -0,0 +1,46 @@ +""" +``_get_httpx_client`` + ``HTTPHandler.post`` (same pattern as Azure Anthropic sync path: +``_get_httpx_client(params={"timeout": ...})`` then ``post(..., timeout=...)``). + +Uses https://httpbin.org/delay/10 with ``timeout=5`` — the handler must raise :class:`~litellm.exceptions.Timeout` +before the 10s delay completes. Skips if httpbin is unreachable. + +Lives under ``local_testing`` (not ``make test-unit``). +""" + +import json +import os +import sys + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +) + +from litellm.exceptions import Timeout as LitellmTimeout +from litellm.llms.custom_httpx.http_handler import _get_httpx_client + +_HTTPBIN_DELAY_S = 10 +_PER_REQUEST_TIMEOUT_S = 5.0 +_CLIENT_DEFAULT_TIMEOUT_S = 60.0 + + +def test_post_delay_exceeds_per_request_timeout_raises(): + try: + httpx.get("https://httpbin.org/get", timeout=5.0) + except Exception as e: + pytest.skip(f"httpbin.org unreachable: {e}") + + handler = _get_httpx_client(params={"timeout": _CLIENT_DEFAULT_TIMEOUT_S}) + try: + with pytest.raises(LitellmTimeout): + handler.post( + f"https://httpbin.org/delay/{_HTTPBIN_DELAY_S}", + headers={"content-type": "application/json"}, + data=json.dumps({"model": "claude", "messages": []}), + timeout=_PER_REQUEST_TIMEOUT_S, + ) + finally: + handler.close() diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_handler.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_handler.py index ddfad420d04..ad86077224a 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_handler.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_handler.py @@ -222,5 +222,7 @@ class TestAzureAnthropicChatCompletion: # Verify non-streaming was handled mock_client.post.assert_called_once() + mock_get_client.assert_called_once_with(params={"timeout": timeout}) + assert mock_client.post.call_args.kwargs["timeout"] == timeout assert result is not None diff --git a/tests/test_litellm/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py b/tests/test_litellm/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py new file mode 100644 index 00000000000..a94034e5107 --- /dev/null +++ b/tests/test_litellm/llms/azure_ai/claude/test_main_azure_anthropic_timeout.py @@ -0,0 +1,42 @@ +""" +Ensure litellm.completion() forwards timeout to Azure Anthropic handler (main.py dispatch). +""" + +import os +import sys +from unittest.mock import MagicMock, patch + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) +) + +from litellm import completion +from litellm.types.utils import ModelResponse + + +def test_main_azure_ai_claude_completion_passes_timeout_to_azure_anthropic_handler(): + captured: dict = {} + + def fake_azure_anthropic_completion(**kwargs): + captured.update(kwargs) + return ModelResponse() + + with patch( + "litellm.main.azure_anthropic_chat_completions" + ) as mock_azure_anthropic: + mock_azure_anthropic.completion = MagicMock( + side_effect=fake_azure_anthropic_completion + ) + + completion( + model="azure_ai/claude-sonnet-4-5", + messages=[{"role": "user", "content": "hi"}], + api_base="https://example.services.ai.azure.com/anthropic", + api_key="test-key", + timeout=42.5, + ) + + mock_azure_anthropic.completion.assert_called_once() + assert captured["timeout"] == 42.5 + assert captured["model"] == "claude-sonnet-4-5" + assert captured["custom_llm_provider"] == "azure_ai" diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 0a3f0fe5e67..5fad8a99083 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -15,7 +15,12 @@ sys.path.insert( ) # Adds the parent directory to the system path import litellm from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_ssl_configuration +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_ssl_configuration, +) @pytest.mark.asyncio @@ -658,3 +663,26 @@ async def test_httpx_handler_uses_env_user_agent(monkeypatch): assert req.headers.get("User-Agent") == "Claude Code" finally: await handler.close() + + +def test_get_httpx_client_applies_float_timeout_without_mocking_handler(): + """ + Exercise real _get_httpx_client + HTTPHandler: params={'timeout': x} must reach httpx.Client(timeout=...). + Uses an uncommon timeout value to avoid colliding with other cached clients in-process. + """ + timeout = 3847.291 + handler = _get_httpx_client(params={"timeout": timeout}) + try: + assert isinstance(handler, HTTPHandler) + assert handler.client.timeout == httpx.Timeout(timeout) + finally: + handler.close() + + +def test_get_httpx_client_applies_httpx_timeout_object_without_mocking_handler(): + t = httpx.Timeout(40.0, connect=5.0) + handler = _get_httpx_client(params={"timeout": t}) + try: + assert handler.client.timeout == t + finally: + handler.close() diff --git a/tests/test_litellm/test_completion_timeout_resolution.py b/tests/test_litellm/test_completion_timeout_resolution.py new file mode 100644 index 00000000000..19cd1cef803 --- /dev/null +++ b/tests/test_litellm/test_completion_timeout_resolution.py @@ -0,0 +1,93 @@ +"""Unit tests for litellm.main._resolve_completion_timeout (completion() timeout chain).""" + +import os +import sys + +import httpx + +sys.path.insert( + 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +) + +import litellm +from litellm.main import _resolve_completion_timeout + + +def test_explicit_timeout_wins(): + assert ( + _resolve_completion_timeout( + timeout=12.5, + kwargs={"timeout": 99.0, "request_timeout": 88.0}, + custom_llm_provider="openai", + ) + == 12.5 + ) + + +def test_kwargs_timeout_when_param_none(): + assert ( + _resolve_completion_timeout( + timeout=None, + kwargs={"timeout": 21.0}, + custom_llm_provider="azure_ai", + ) + == 21.0 + ) + + +def test_request_timeout_alias_in_kwargs(): + assert ( + _resolve_completion_timeout( + timeout=None, + kwargs={"request_timeout": 33.0}, + custom_llm_provider="bedrock", + ) + == 33.0 + ) + + +def test_litellm_module_request_timeout(monkeypatch): + monkeypatch.setattr(litellm, "request_timeout", 360.0) + assert ( + _resolve_completion_timeout( + timeout=None, + kwargs={}, + custom_llm_provider="vertex_ai", + ) + == 360.0 + ) + + +def test_fallback_600_when_no_timeout_anywhere(monkeypatch): + """600 applies only when named, kwargs, and litellm.request_timeout are all unset.""" + monkeypatch.setattr(litellm, "request_timeout", None) + assert ( + _resolve_completion_timeout( + timeout=None, + kwargs={}, + custom_llm_provider="azure_ai", + ) + == 600.0 + ) + + +def test_httpx_timeout_coerced_for_provider_without_httpx_timeout_support(): + t = httpx.Timeout(50.0, connect=2.0) + out = _resolve_completion_timeout( + timeout=t, + kwargs={}, + custom_llm_provider="azure_ai", + ) + assert out == 50.0 + assert not isinstance(out, httpx.Timeout) + + +def test_httpx_timeout_preserved_for_openai(): + t = httpx.Timeout(40.0, connect=5.0) + out = _resolve_completion_timeout( + timeout=t, + kwargs={}, + custom_llm_provider="openai", + ) + assert out is t + assert isinstance(out, httpx.Timeout)