mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
d64ac3998c
* feat(router): let a deployment set a timeout per service_tier
Adds timeout_by_service_tier and stream_timeout_by_service_tier to a deployment's
litellm_params. A request whose service_tier matches a key uses that value in place
of the deployment's timeout or stream_timeout. A timeout the client sends still wins
and requests without a matching tier are unchanged
Co-authored-by: adc77 <amandogra2016@gmail.com>
* style(router): sort the request_timeout_resolver import
* fix(router): read the service tier maps without the untyped params get
* fix(router): look the service tier timeout up through a typed get
---------
Backport of #47406 to stable/1.104.x
Cherry-picked from 62ce515d15 (main)
Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: adc77 <amandogra2016@gmail.com>
87 lines
3.4 KiB
Python
87 lines
3.4 KiB
Python
import sys
|
|
from collections.abc import Mapping
|
|
from types import MappingProxyType
|
|
from typing import Final
|
|
|
|
from pydantic import TypeAdapter
|
|
|
|
from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout
|
|
|
|
DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0
|
|
|
|
_SECONDS: Final = TypeAdapter(float)
|
|
_NO_PARAMS: Final[Mapping[str, object]] = MappingProxyType({})
|
|
|
|
|
|
def resolve_pass_through_request_timeout(
|
|
endpoint_timeout: float | None = None,
|
|
) -> float:
|
|
"""
|
|
Resolve the upstream httpx timeout for pass_through_request.
|
|
|
|
Precedence: per-endpoint timeout -> general_settings.pass_through_request_timeout -> 600s default.
|
|
|
|
Uses sys.modules to read general_settings only when the proxy module is already
|
|
loaded, avoiding a fastapi transitive import in pure SDK contexts.
|
|
"""
|
|
if endpoint_timeout is not None:
|
|
return float(endpoint_timeout)
|
|
|
|
try:
|
|
proxy_server: Final = sys.modules.get("litellm.proxy.proxy_server")
|
|
if proxy_server is not None:
|
|
global_timeout: Final = getattr(proxy_server, "general_settings", {}).get("pass_through_request_timeout")
|
|
if global_timeout is not None:
|
|
return float(global_timeout)
|
|
except Exception:
|
|
pass
|
|
|
|
return DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS
|
|
|
|
|
|
def resolve_llm_passthrough_timeout(
|
|
kwargs: Mapping[str, object] | None = None,
|
|
litellm_params: Mapping[str, object] | None = None,
|
|
router_timeout: float | str | None = None,
|
|
router_stream_timeout: float | str | None = None,
|
|
) -> float:
|
|
"""
|
|
Resolve upstream httpx timeout for SDK native passthrough (e.g. Bedrock /converse,
|
|
Anthropic /v1/messages).
|
|
|
|
Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params
|
|
timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout
|
|
-> general_settings.pass_through_request_timeout -> 600s default.
|
|
|
|
Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before
|
|
any generic timeout, matching ``Router._get_stream_timeout`` on the completion route:
|
|
kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] ->
|
|
litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above.
|
|
|
|
Only the first set value is validated as seconds, so a value in a lower-precedence
|
|
field never fails the call.
|
|
"""
|
|
request: Final = kwargs if kwargs is not None else _NO_PARAMS
|
|
deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS
|
|
service_tier: Final = request.get("service_tier")
|
|
stream_candidates: Final = (
|
|
(
|
|
request.get("stream_timeout"),
|
|
get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"),
|
|
deployment.get("stream_timeout"),
|
|
router_stream_timeout,
|
|
)
|
|
if request.get("stream")
|
|
else ()
|
|
)
|
|
candidates: Final = (
|
|
*stream_candidates,
|
|
request.get("timeout"),
|
|
request.get("request_timeout"),
|
|
get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"),
|
|
deployment.get("timeout"),
|
|
deployment.get("request_timeout"),
|
|
router_timeout,
|
|
)
|
|
winner: Final = next((val for val in candidates if val is not None), None)
|
|
return resolve_pass_through_request_timeout() if winner is None else _SECONDS.validate_python(winner)
|