From 0fa9cd14267d3c564d5368076f5bbdab5b2c1e4b Mon Sep 17 00:00:00 2001 From: harish876 Date: Tue, 14 Apr 2026 03:53:32 +0000 Subject: [PATCH] update code structure, test cases --- litellm/constants.py | 2 +- .../litellm_core_utils/completion_timeout.py | 56 +++++++++ litellm/main.py | 76 ++---------- .../test_completion_timeout_resolution.py | 111 ++++++++++++------ 4 files changed, 139 insertions(+), 106 deletions(-) create mode 100644 litellm/litellm_core_utils/completion_timeout.py diff --git a/litellm/constants.py b/litellm/constants.py index 3318bbba50a..1703102bc84 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -417,7 +417,7 @@ DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2 # value so longer-running surfaces (Router `timeout or litellm.request_timeout`, # speech/TTS, responses, vector stores, etc.) get a long HTTP deadline. Chat # `completion()` maps this sentinel down to 600s when the caller did not set a -# per-request/model timeout—see `_resolve_completion_timeout` in main.py. MCP uses +# per-request/model timeout—see ``CompletionTimeout.resolve`` in completion_timeout.py. MCP uses # dedicated timeouts (e.g. `MCP_CLIENT_TIMEOUT`), not `request_timeout`. DEFAULT_REQUEST_TIMEOUT_SECONDS: float = 6000.0 request_timeout: float = float( diff --git a/litellm/litellm_core_utils/completion_timeout.py b/litellm/litellm_core_utils/completion_timeout.py new file mode 100644 index 00000000000..e1ece9170e2 --- /dev/null +++ b/litellm/litellm_core_utils/completion_timeout.py @@ -0,0 +1,56 @@ +"""Completion HTTP timeout resolution (kept out of ``main.py`` to limit import cycles).""" + +from __future__ import annotations + +from typing import Callable, Optional, Union + +import httpx + +from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS + + +class CompletionTimeout: + """Resolves HTTP timeout for ``completion()`` from model vs global settings.""" + + @staticmethod + def resolve( + model_timeout: Optional[Union[float, str, httpx.Timeout]], + kwargs: dict, + custom_llm_provider: str, + *, + global_timeout: Optional[Union[float, str, httpx.Timeout]], + supports_httpx_timeout: Callable[[str], bool], + ) -> Union[float, httpx.Timeout]: + """ + Order: ``model_timeout`` (call argument / merged ``litellm_params``), then + ``kwargs["timeout"]``, ``kwargs["request_timeout"]``, then ``global_timeout`` + (e.g. :attr:`litellm.request_timeout` from proxy ``litellm_settings``), else ``600``. + + Coerce :class:`httpx.Timeout` when the provider does not support it; map ``6000`` + (:data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS`) to ``600`` for completion. + """ + if model_timeout is not None: + resolved: Union[float, str, httpx.Timeout] = model_timeout + elif kwargs.get("timeout") is not None: + resolved = kwargs["timeout"] + elif kwargs.get("request_timeout") is not None: + resolved = kwargs["request_timeout"] + else: + resolved = global_timeout if global_timeout is not None else 600 + + if isinstance(resolved, httpx.Timeout) and not supports_httpx_timeout( + custom_llm_provider + ): + read_timeout = resolved.read + resolved = ( + float(read_timeout) if read_timeout is not None else 600.0 + ) # default 10 min timeout + elif not isinstance(resolved, httpx.Timeout): + resolved = float(resolved) # type: ignore + + if not isinstance(resolved, httpx.Timeout) and float(resolved) == float( + DEFAULT_REQUEST_TIMEOUT_SECONDS + ): + resolved = 600.0 + + return resolved diff --git a/litellm/main.py b/litellm/main.py index bb467002ec7..6508aeec914 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -68,7 +68,6 @@ if TYPE_CHECKING: from litellm.constants import ( DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT, DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, - DEFAULT_REQUEST_TIMEOUT_SECONDS, ) from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger @@ -77,6 +76,7 @@ from litellm.litellm_core_utils.audio_utils.utils import ( calculate_request_duration, get_audio_file_for_health_check, ) +from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_provider_specific_headers import ( ProviderSpecificHeaderUtils, @@ -1046,70 +1046,6 @@ def _build_custom_pricing_entry( return entry -def _resolve_completion_timeout( - timeout: Optional[Union[float, str, httpx.Timeout]], - kwargs: dict, - custom_llm_provider: str, -) -> Union[float, httpx.Timeout]: - """ - Resolve timeout inside completion(). - - Sources (first match wins): - - - **Model / deployment config:** the `timeout` argument (e.g. from router merging - per-model `litellm_params`, including a deployment-level `timeout`). - - **Model config alias:** ``kwargs["request_timeout"]`` when the caller passes the - per-model ``request_timeout`` field from model config (same idea as deployment - `litellm_params.request_timeout`). - - **Global module default:** :attr:`litellm.request_timeout` (from - ``litellm_settings.request_timeout`` on the proxy when set, otherwise - :data:`~litellm.constants.DEFAULT_REQUEST_TIMEOUT_SECONDS`, i.e. ``6000`` seconds). - That long default is shared with Router, speech/TTS, and other subsystems; for - chat completion only, if the timeout came solely from this module attribute and - still equals ``6000``, it is treated as unset and ``600`` seconds is used instead. - - - **Fallback:** ``600`` seconds if no timeout is resolved above. - - Also accepts ``kwargs["timeout"]`` as a fallback when the named ``timeout`` argument - is omitted. - - If the resolved value is :class:`httpx.Timeout` and the provider does not support - passing it through (:func:`litellm.utils.supports_httpx_timeout`), coerce to a - float (read timeout, or ``600.0`` if read is unset). Otherwise numeric strings / - floats are coerced with ``float(...)``. - """ - if timeout is None: - timeout = kwargs.get("timeout") - if timeout is None: - timeout = kwargs.get("request_timeout") - resolved_from_litellm_request_timeout_attr = False - if timeout is None: - timeout = getattr(litellm, "request_timeout", None) - if timeout is not None: - resolved_from_litellm_request_timeout_attr = True - if timeout is None: - timeout = 600 - elif ( - resolved_from_litellm_request_timeout_attr - and not isinstance(timeout, httpx.Timeout) - and float(timeout) == float(DEFAULT_REQUEST_TIMEOUT_SECONDS) - ): - # 6000s is the package default for litellm.request_timeout so MCP, speech/TTS, - # Router, and similar paths keep a long deadline. completion() uses 600s when - # nothing more specific was supplied (explicit kwargs still win above). - timeout = 600 - if isinstance(timeout, httpx.Timeout) and not supports_httpx_timeout( - custom_llm_provider - ): - read_timeout = timeout.read - timeout = ( - float(read_timeout) if read_timeout is not None else 600.0 - ) # default 10 min timeout - elif not isinstance(timeout, httpx.Timeout): - timeout = float(timeout) # type: ignore - return timeout - - @tracer.wrap() @client def completion( # type: ignore # noqa: PLR0915 @@ -1465,10 +1401,12 @@ def completion( # type: ignore # noqa: PLR0915 ) # support region-based pricing for bedrock ### TIMEOUT LOGIC ### - timeout = _resolve_completion_timeout( - timeout=timeout, - kwargs=kwargs, - custom_llm_provider=custom_llm_provider, + timeout = CompletionTimeout.resolve( + timeout, + kwargs, + custom_llm_provider, + global_timeout=getattr(litellm, "request_timeout", None), + supports_httpx_timeout=supports_httpx_timeout, ) ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### diff --git a/tests/test_litellm/test_completion_timeout_resolution.py b/tests/test_litellm/test_completion_timeout_resolution.py index 19cd1cef803..f2955852afe 100644 --- a/tests/test_litellm/test_completion_timeout_resolution.py +++ b/tests/test_litellm/test_completion_timeout_resolution.py @@ -1,4 +1,4 @@ -"""Unit tests for litellm.main._resolve_completion_timeout (completion() timeout chain).""" +"""Unit tests for litellm.litellm_core_utils.completion_timeout.CompletionTimeout.""" import os import sys @@ -9,16 +9,18 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) ) -import litellm -from litellm.main import _resolve_completion_timeout +from litellm.litellm_core_utils.completion_timeout import CompletionTimeout +from litellm.utils import supports_httpx_timeout def test_explicit_timeout_wins(): assert ( - _resolve_completion_timeout( - timeout=12.5, - kwargs={"timeout": 99.0, "request_timeout": 88.0}, - custom_llm_provider="openai", + CompletionTimeout.resolve( + 12.5, + {"timeout": 99.0, "request_timeout": 88.0}, + "openai", + global_timeout=None, + supports_httpx_timeout=supports_httpx_timeout, ) == 12.5 ) @@ -26,10 +28,12 @@ def test_explicit_timeout_wins(): def test_kwargs_timeout_when_param_none(): assert ( - _resolve_completion_timeout( - timeout=None, - kwargs={"timeout": 21.0}, - custom_llm_provider="azure_ai", + CompletionTimeout.resolve( + None, + {"timeout": 21.0}, + "azure_ai", + global_timeout=None, + supports_httpx_timeout=supports_httpx_timeout, ) == 21.0 ) @@ -37,35 +41,66 @@ def test_kwargs_timeout_when_param_none(): def test_request_timeout_alias_in_kwargs(): assert ( - _resolve_completion_timeout( - timeout=None, - kwargs={"request_timeout": 33.0}, - custom_llm_provider="bedrock", + CompletionTimeout.resolve( + None, + {"request_timeout": 33.0}, + "bedrock", + global_timeout=None, + supports_httpx_timeout=supports_httpx_timeout, ) == 33.0 ) -def test_litellm_module_request_timeout(monkeypatch): - monkeypatch.setattr(litellm, "request_timeout", 360.0) +def test_global_timeout_from_litellm_settings(): assert ( - _resolve_completion_timeout( - timeout=None, - kwargs={}, - custom_llm_provider="vertex_ai", + CompletionTimeout.resolve( + None, + {}, + "vertex_ai", + global_timeout=360.0, + supports_httpx_timeout=supports_httpx_timeout, ) == 360.0 ) -def test_fallback_600_when_no_timeout_anywhere(monkeypatch): - """600 applies only when named, kwargs, and litellm.request_timeout are all unset.""" - monkeypatch.setattr(litellm, "request_timeout", None) +def test_global_timeout_package_default_coerced_to_600_for_completion(): + """Package default 6000s → 600s for completion-only path.""" assert ( - _resolve_completion_timeout( - timeout=None, - kwargs={}, - custom_llm_provider="azure_ai", + CompletionTimeout.resolve( + None, + {}, + "openai", + global_timeout=6000.0, + supports_httpx_timeout=supports_httpx_timeout, + ) + == 600.0 + ) + + +def test_explicit_request_timeout_6000_normalized_to_completion_default(): + """6000 is the package sentinel; completion always uses 600 instead.""" + assert ( + CompletionTimeout.resolve( + None, + {"request_timeout": 6000.0}, + "openai", + global_timeout=None, + supports_httpx_timeout=supports_httpx_timeout, + ) + == 600.0 + ) + + +def test_fallback_600_when_no_global_timeout(): + assert ( + CompletionTimeout.resolve( + None, + {}, + "azure_ai", + global_timeout=None, + supports_httpx_timeout=supports_httpx_timeout, ) == 600.0 ) @@ -73,10 +108,12 @@ def test_fallback_600_when_no_timeout_anywhere(monkeypatch): def test_httpx_timeout_coerced_for_provider_without_httpx_timeout_support(): t = httpx.Timeout(50.0, connect=2.0) - out = _resolve_completion_timeout( - timeout=t, - kwargs={}, - custom_llm_provider="azure_ai", + out = CompletionTimeout.resolve( + t, + {}, + "azure_ai", + global_timeout=None, + supports_httpx_timeout=supports_httpx_timeout, ) assert out == 50.0 assert not isinstance(out, httpx.Timeout) @@ -84,10 +121,12 @@ def test_httpx_timeout_coerced_for_provider_without_httpx_timeout_support(): def test_httpx_timeout_preserved_for_openai(): t = httpx.Timeout(40.0, connect=5.0) - out = _resolve_completion_timeout( - timeout=t, - kwargs={}, - custom_llm_provider="openai", + out = CompletionTimeout.resolve( + t, + {}, + "openai", + global_timeout=None, + supports_httpx_timeout=supports_httpx_timeout, ) assert out is t assert isinstance(out, httpx.Timeout)