scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 267

feat(router): let a deployment set a timeout per service_tier (#47406)

Browse source 
* feat(router): let a deployment set a timeout per service_tier

Adds timeout_by_service_tier and stream_timeout_by_service_tier to a deployment's
litellm_params. A request whose service_tier matches a key uses that value in place
of the deployment's timeout or stream_timeout. A timeout the client sends still wins
and requests without a matching tier are unchanged

Co-authored-by: adc77 <amandogra2016@gmail.com>

* style(router): sort the request_timeout_resolver import

* fix(router): read the service tier maps without the untyped params get

* fix(router): look the service tier timeout up through a typed get

---------

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: adc77 <amandogra2016@gmail.com>

Backport of #47406 to stable/1.105.x.
Cherry-picked from 62ce515d15 (main).
This commit is contained in:
devin-ai-integration[bot] 2026-10-20 23:56:01 +00:00
parent 7f233c62c4
commit 2e1879d8cd
9 changed files with 148 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
litellm/litellm_core_utils/request_timeout_resolver.py
View file

@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that:
from __future__ import annotations
from typing import Final
from collections.abc import Mapping
from typing import Final, Literal, Protocol
from pydantic import TypeAdapter
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None)
def get_configured_request_timeout() -> float | None:
"""Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None:
if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
return timeout
return None
class _DeploymentParams(Protocol):
def get(self, key: str, /) -> object: ...
def get_service_tier_timeout(
service_tier: object,
litellm_params: _DeploymentParams,
key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"],
) -> float | None:
"""Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``."""
if not isinstance(service_tier, str):
return None
timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key))
return None if timeouts is None else timeouts.get(service_tier)

20
litellm/passthrough/timeout_utils.py
View file

@ -5,7 +5,10 @@ from typing import Final
from pydantic import TypeAdapter
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
get_service_tier_timeout,
)
DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0
@ -50,21 +53,27 @@ def resolve_llm_passthrough_timeout(
Anthropic /v1/messages).
Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params
timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout,
timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout,
when explicitly set) -> general_settings.pass_through_request_timeout -> 600s default.
Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before
any generic timeout, matching ``Router._get_stream_timeout`` on the completion route:
kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the
non-streaming chain above.
kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] ->
litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above.
Only the first set value is validated as seconds, so a value in a lower-precedence
field never fails the call.
"""
request: Final = kwargs if kwargs is not None else _NO_PARAMS
deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS
service_tier: Final = request.get("service_tier")
stream_candidates: Final = (
(request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout)
(
request.get("stream_timeout"),
get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"),
deployment.get("stream_timeout"),
router_stream_timeout,
)
if request.get("stream")
else ()
)
@ -72,6 +81,7 @@ def resolve_llm_passthrough_timeout(
*stream_candidates,
request.get("timeout"),
request.get("request_timeout"),
get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"),
deployment.get("timeout"),
deployment.get("request_timeout"),
router_timeout,

3
litellm/router.py
View file

@ -99,6 +99,7 @@ from litellm.litellm_core_utils.ptu_pricing import (
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
get_service_tier_timeout,
)
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.sensitive_data_masker import (
@ -4123,6 +4124,7 @@ class Router:
"""Helper to get stream timeout from kwargs or deployment params"""
return (
kwargs.get("stream_timeout", None) # the params dynamically set by user
or get_service_tier_timeout(kwargs.get("service_tier"), data, "stream_timeout_by_service_tier")
or data.get("stream_timeout", None) # timeout set on litellm_params for this deployment
or self.stream_timeout # timeout set on router
or self.request_timeout # litellm_settings.request_timeout (per-attempt)
@ -4134,6 +4136,7 @@ class Router:
timeout: Final = (
kwargs.get("timeout", None) # the params dynamically set by user
or kwargs.get("request_timeout", None) # the params dynamically set by user
or get_service_tier_timeout(kwargs.get("service_tier"), data, "timeout_by_service_tier")
or data.get("timeout", None) # timeout set on litellm_params for this deployment
or data.get("request_timeout", None) # timeout set on litellm_params for this deployment
or self.request_timeout # litellm_settings.request_timeout (per-attempt)

2
litellm/types/litellm_params.py
View file

@ -76,6 +76,8 @@ class ProviderConnection:
request_timeout: float | None = None
force_timeout: float | None = None
stream_timeout: float | str | None = None
timeout_by_service_tier: Mapping[str, float] | None = None
stream_timeout_by_service_tier: Mapping[str, float] | None = None
max_retries: int | None = None
tenant_id: str | None = None
client_id: str | None = None

5
litellm/types/router.py
View file

@ -390,6 +390,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
otpm: int | None = None
timeout: float | str | httpx.Timeout | None = None # if str, pass in as os.environ/
stream_timeout: float | str | None = None # timeout when making stream=True calls, if str, pass in as os.environ/
# per service_tier overrides of timeout / stream_timeout, used when the request's service_tier matches a key
timeout_by_service_tier: Mapping[str, float] | None = None
stream_timeout_by_service_tier: Mapping[str, float] | None = None
max_retries: int | None = None
drop_params: bool | str | None = None
organization: str | None = None # for openai orgs
@ -574,6 +577,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
api_version: str | None
timeout: float | str | httpx.Timeout | None
stream_timeout: float | str | None
timeout_by_service_tier: ReadOnly[Mapping[str, float] | None]
stream_timeout_by_service_tier: ReadOnly[Mapping[str, float] | None]
max_retries: int | None
organization: list | str | None # for openai orgs
provider_affinity_header: ReadOnly[str | None]

57
tests/local_testing/test_router_timeout.py
View file

@ -5,6 +5,7 @@ import asyncio
import os
import time
import traceback
from typing import Final
import pytest
@ -337,3 +338,59 @@ def test_unit_test_streaming_timeout(stream):
assert stream_timeout_val == stream_timeout
else:
assert stream_timeout_val == normal_timeout
@pytest.mark.parametrize(
"request_kwargs, expected_timeout",
[
({"service_tier": "flex"}, 900.0),
({"service_tier": "default"}, 610.0),
({}, 610.0),
({"service_tier": "flex", "timeout": 20}, 20),
({"service_tier": "flex", "stream": True}, 950.0),
({"service_tier": "default", "stream": True}, 610.0),
({"service_tier": "flex", "stream": True, "stream_timeout": 30}, 30),
],
)
def test_the_deployment_timeout_for_the_requests_service_tier_wins_over_its_general_timeout(
request_kwargs: dict[str, object], expected_timeout: float
) -> None:
router: Final = Router(
model_list=[
{
"model_name": "openai-gpt-5.1",
"litellm_params": {
"model": "openai/gpt-5.1",
"api_key": "sk-test",
"timeout": 610,
"stream_timeout": 610,
"timeout_by_service_tier": {"flex": 900},
"stream_timeout_by_service_tier": {"flex": 950},
},
}
]
)
deployment_params: Final = router.model_list[0]["litellm_params"]
assert router._get_timeout(kwargs=request_kwargs, data=deployment_params) == expected_timeout
def test_a_stream_keeps_the_deployment_stream_timeout_when_only_the_request_timeout_map_names_its_tier() -> None:
router: Final = Router(
model_list=[
{
"model_name": "openai-gpt-5.1",
"litellm_params": {
"model": "openai/gpt-5.1",
"api_key": "sk-test",
"timeout": 610,
"stream_timeout": 30,
"timeout_by_service_tier": {"flex": 900},
},
}
]
)
deployment_params: Final = router.model_list[0]["litellm_params"]
assert router._get_timeout(kwargs={"service_tier": "flex", "stream": True}, data=deployment_params) == 30.0
assert router._get_timeout(kwargs={"service_tier": "flex"}, data=deployment_params) == 900.0

26
tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py
View file

@ -1278,6 +1278,32 @@ def test_resolve_llm_passthrough_timeout_validates_only_the_winning_value(
assert resolve_llm_passthrough_timeout(kwargs=kwargs, litellm_params=litellm_params) == expected
def test_resolve_llm_passthrough_timeout_uses_the_deployment_timeout_for_the_requests_service_tier():
litellm_params = {
"timeout": 610,
"stream_timeout": 30,
"timeout_by_service_tier": {"flex": 900},
"stream_timeout_by_service_tier": {"flex": 950},
}
assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex"}, litellm_params=litellm_params) == 900.0
assert resolve_llm_passthrough_timeout(kwargs={"service_tier": "default"}, litellm_params=litellm_params) == 610.0
assert (
resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "timeout": 20}, litellm_params=litellm_params)
== 20.0
)
assert (
resolve_llm_passthrough_timeout(kwargs={"service_tier": "flex", "stream": True}, litellm_params=litellm_params)
== 950.0
)
assert (
resolve_llm_passthrough_timeout(
kwargs={"service_tier": "flex", "stream": True},
litellm_params={"timeout": 610, "stream_timeout": 30, "timeout_by_service_tier": {"flex": 900}},
)
== 30.0
)
def test_resolve_llm_passthrough_timeout_rejects_a_non_numeric_winner():
with pytest.raises(ValidationError):
resolve_llm_passthrough_timeout(kwargs={"timeout": httpx.Timeout(30.0)})

2
tests/unit/types/test_litellm_params.py
View file

@ -74,6 +74,8 @@ CONNECTION_NAMES: Final = (
"request_timeout",
"force_timeout",
"stream_timeout",
"timeout_by_service_tier",
"stream_timeout_by_service_tier",
"max_retries",
"tenant_id",
"client_id",

16
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -34912,6 +34912,10 @@ export interface components {
} | null;
/** Stream Timeout */
stream_timeout?: number | string | null;
/** Stream Timeout By Service Tier */
stream_timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tag Regex */
tag_regex?: string[] | null;
/** Tags */
@ -34924,6 +34928,10 @@ export interface components {
}[] | null;
/** Timeout */
timeout?: number | string | null;
/** Timeout By Service Tier */
timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tpm */
tpm?: number | null;
/** Use Chat Completions Api */
@ -49394,6 +49402,10 @@ export interface components {
} | null;
/** Stream Timeout */
stream_timeout?: number | string | null;
/** Stream Timeout By Service Tier */
stream_timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tag Regex */
tag_regex?: string[] | null;
/** Tags */
@ -49406,6 +49418,10 @@ export interface components {
}[] | null;
/** Timeout */
timeout?: number | string | null;
/** Timeout By Service Tier */
timeout_by_service_tier?: {
[key: string]: number;
} | null;
/** Tpm */
tpm?: number | null;
/** Use Chat Completions Api */