scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 267

feat(router): let a deployment set a timeout per service_tier (#47406)

Browse source 
Backport of #47406 to stable/1.103.x

Cherry-picked from 62ce515d15 (main)
This commit is contained in:
devin-ai-integration[bot] 2026-10-21 00:01:46 +00:00
parent 8147fae4d6
commit 37b7c14830
7 changed files with 146 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
litellm/litellm_core_utils/request_timeout_resolver.py
View file

@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that:
from __future__ import annotations
from typing import Final
from collections.abc import Mapping
from typing import Final, Literal, Protocol
from pydantic import TypeAdapter
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None)
def get_configured_request_timeout() -> float | None:
"""Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None:
if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
return timeout
return None
class _DeploymentParams(Protocol):
def get(self, key: str, /) -> object: ...
def get_service_tier_timeout(
service_tier: object,
litellm_params: _DeploymentParams,
key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"],
) -> float | None:
"""Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``."""
if not isinstance(service_tier, str):
return None
timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key))
return None if timeouts is None else timeouts.get(service_tier)

22
litellm/passthrough/timeout_utils.py
View file

@ -5,6 +5,11 @@ from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
get_service_tier_timeout,
)
DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0
_SECONDS: Final = TypeAdapter(float)
@ -48,21 +53,27 @@ def resolve_llm_passthrough_timeout(
Anthropic /v1/messages).
Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params
timeout/request_timeout -> router_timeout -> general_settings.pass_through_request_timeout
-> 600s default.
timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout,
when explicitly set) -> general_settings.pass_through_request_timeout -> 600s default.
Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before
any generic timeout, matching ``Router._get_stream_timeout`` on the completion route:
kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the
non-streaming chain above.
kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] ->
litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above.
Only the first set value is validated as seconds, so a value in a lower-precedence
field never fails the call.
"""
request: Final = kwargs if kwargs is not None else _NO_PARAMS
deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS
service_tier: Final = request.get("service_tier")
stream_candidates: Final = (
(request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout)
(
request.get("stream_timeout"),
get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"),
deployment.get("stream_timeout"),
router_stream_timeout,
)
if request.get("stream")
else ()
)
@ -70,6 +81,7 @@ def resolve_llm_passthrough_timeout(
*stream_candidates,
request.get("timeout"),
request.get("request_timeout"),
get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"),
deployment.get("timeout"),
deployment.get("request_timeout"),
router_timeout,

3
litellm/router.py
View file

@ -96,6 +96,7 @@ from litellm.litellm_core_utils.ptu_pricing import (
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
get_service_tier_timeout,
)
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.sensitive_data_masker import (
@ -3996,6 +3997,7 @@ class Router:
"""Helper to get stream timeout from kwargs or deployment params"""
return (
kwargs.get("stream_timeout", None) # the params dynamically set by user
or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier")
or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment
or self.stream_timeout # timeout set on router
or self.request_timeout # litellm_settings.request_timeout (per-attempt)
@ -4007,6 +4009,7 @@ class Router:
timeout: Final = (
kwargs.get("timeout", None) # the params dynamically set by user
or kwargs.get("request_timeout", None) # the params dynamically set by user
or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier")
or data.get("timeout", None) # timeout set on litellm_params for this deployment
or data.get("request_timeout", None) # timeout set on litellm_params for this deployment
or self.request_timeout # litellm_settings.request_timeout (per-attempt)

5
litellm/types/router.py
View file

@ -327,6 +327,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
otpm: int | None = None
timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/
stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/
# per service_tier overrides of timeout / stream_timeout, used when the request's service_tier matches a key
timeout_by_service_tier: Mapping[str, float] | None = None
stream_timeout_by_service_tier: Mapping[str, float] | None = None
max_retries: int | None = None
drop_params: bool | str | None = None
organization: str | None = None # for openai orgs
@ -503,6 +506,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
api_version: str | None
timeout: float | str | httpx.Timeout | None
stream_timeout: float | str | None
timeout_by_service_tier: ReadOnly[Mapping[str, float] | None]
stream_timeout_by_service_tier: ReadOnly[Mapping[str, float] | None]
max_retries: int | None
organization: list | str | None # for openai orgs
configurable_clientside_auth_params: (

57
tests/local_testing/test_router_timeout.py
View file

@ -7,6 +7,7 @@ import time
import traceback
import pytest
from typing import Final
from unittest.mock import patch, MagicMock, AsyncMock
@ -337,3 +338,59 @@ def test_unit_test_streaming_timeout(stream):
assert stream_timeout_val == stream_timeout
else:
assert stream_timeout_val == normal_timeout
@pytest.mark.parametrize(
"request_kwargs, expected_timeout",
[
({"service_tier": "flex"}, 900.0),
({"service_tier": "default"}, 610.0),
({}, 610.0),
({"service_tier": "flex", "timeout": 20}, 20),
({"service_tier": "flex", "stream": True}, 950.0),
({"service_tier": "default", "stream": True}, 610.0),
({"service_tier": "flex", "stream": True, "stream_timeout": 30}, 30),
],
)
def test_the_deployment_timeout_for_the_requests_service_tier_wins_over_its_general_timeout(
request_kwargs: dict[str, object], expected_timeout: float
) -> None:
router: Final = Router(
model_list=[
{
"model_name": "openai-gpt-5.1",
"litellm_params": {
"model": "openai/gpt-5.1",
"api_key": "sk-test",
"timeout": 610,
"stream_timeout": 610,
"timeout_by_service_tier": {"flex": 900},
"stream_timeout_by_service_tier": {"flex": 950},
},
}
]
)
deployment_params: Final = router.model_list[0]["litellm_params"]
assert router._get_timeout(kwargs=request_kwargs, data=deployment_params) == expected_timeout
def test_a_stream_keeps_the_deployment_stream_timeout_when_only_the_request_timeout_map_names_its_tier() -> None:
router: Final = Router(
model_list=[
{
"model_name": "openai-gpt-5.1",
"litellm_params": {
"model": "openai/gpt-5.1",
"api_key": "sk-test",
"timeout": 610,
"stream_timeout": 30,
"timeout_by_service_tier": {"flex": 900},
},
}
]
)
deployment_params: Final = router.model_list[0]["litellm_params"]
assert router._get_timeout(kwargs={"service_tier": "flex", "stream": True}, data=deployment_params) == 30.0
assert router._get_timeout(kwargs={"service_tier": "flex"}, data=deployment_params) == 900.0

26
tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -1243,6 +1243,32 @@ def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value(
assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected
def test_resolve_llm_passthrough_timeout_uses_the_deployment_timeout_for_the_requests_service_tier():
litellm_params = {
"timeout": 610,
"stream_timeout": 30,
"timeout_by_service_tier": {"flex": 900},
"stream_timeout_by_service_tier": {"flex": 950},
}
assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex"}, litellm_params=litellm_params) == 900.0
assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "default"}, litellm_params=litellm_params) == 610.0
assert (
resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "timeout": 20}, litellm_params=litellm_params)
== 20.0
)
assert (
resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "stream": True}, litellm_params=litellm_params)
== 950.0
)
assert (
resolve_llm_passthrough_timeout(
kwargs={"service_tier": "flex", "stream": True},
litellm_params={"timeout": 610, "stream_timeout": 30, "timeout_by_service_tier": {"flex": 900}},
)
== 30.0
)
def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner():
with pytest.raises(ValidationError):
resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)})

16
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -31133,6 +31133,10 @@ export interface components {
} | null;
/** Stream Timeout */
stream_timeout?: number | string | null;
/** Stream Timeout By Service Tier */
stream_timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tag Regex */
tag_regex?: string[] | null;
/** Tags */
@ -31145,6 +31149,10 @@ export interface components {
}[] | null;
/** Timeout */
timeout?: number | string | null;
/** Timeout By Service Tier */
timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tpm */
tpm?: number | null;
/** Use Chat Completions Api */
@ -41848,6 +41856,10 @@ export interface components {
} | null;
/** Stream Timeout */
stream_timeout?: number | string | null;
/** Stream Timeout By Service Tier */
stream_timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tag Regex */
tag_regex?: string[] | null;
/** Tags */
@ -41860,6 +41872,10 @@ export interface components {
}[] | null;
/** Timeout */
timeout?: number | string | null;
/** Timeout By Service Tier */
timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tpm */
tpm?: number | null;
/** Use Chat Completions Api */