This commit is contained in:
musi22 2026-10-25 18:03:37 +00:00 • committed by GitHub
commit 46b347a071
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 402 additions and 11 deletions

View file

@ -151,6 +151,7 @@ DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
DEFAULT_MAX_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_MAX_COOLDOWN_TIME_SECONDS", 3600)
DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = get_env_float("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", 1.0)
DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLING_RETRIES", 5)
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)

View file

@ -2492,11 +2492,13 @@ async def add_litellm_data_to_request(
verbose_proxy_logger.debug("Request Headers: %s", _logging_safe_headers)
verbose_proxy_logger.debug("Raw Headers: %s", _raw_headers)
if forward_llm_auth and "x-api-key" in _headers:
data["api_key"] = _headers["x-api-key"]
verbose_proxy_logger.debug(
"Setting client-provided x-api-key as api_key parameter (will override deployment key)"
)
if forward_llm_auth:
data["has_forwarded_client_auth"] = True
if "x-api-key" in _headers:
data["api_key"] = _headers["x-api-key"]
verbose_proxy_logger.debug(
"Setting client-provided x-api-key as api_key parameter (will override deployment key)"
)
##########################################################
# Init - Proxy Server Request

View file

@ -69,6 +69,7 @@ from litellm.constants import (
DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
DEFAULT_HEALTH_CHECK_INTERVAL,
DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
DEFAULT_MAX_COOLDOWN_TIME_SECONDS,
DEFAULT_MAX_LRU_CACHE_SIZE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
KUBERNETES_POD_ROUTING_KEY,
@ -240,6 +241,7 @@ from litellm.router_utils.cooldown_handlers import (
is_background_response_poll_not_found,
is_caller_scoped_auth_failure,
is_caller_timeout_408,
is_client_credential_rate_limit,
is_video_lookup_not_found,
set_cooldown_deployments,
)
@ -1226,6 +1228,7 @@ class Router:
allowed_fails: int | None = None, # Number of times a deployment can failbefore being added to cooldown
allowed_fails_policy: AllowedFailsPolicy | None = None, # set custom allowed fails policy
cooldown_time: float | None = None, # (seconds) time to cooldown a deployment after failure
max_cooldown_time: float | None = None, # (seconds) maximum cap for response-header cooldowns
disable_cooldowns: bool | None = None,
fallback_on_unsupported_params: bool = True,
routing_strategy: RoutingStrategyName = "simple-shuffle",
@ -1491,7 +1494,8 @@ class Router:
self.allowed_fails = allowed_fails
else:
self.allowed_fails = litellm.allowed_fails
self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
self.cooldown_time = cooldown_time if cooldown_time is not None else DEFAULT_COOLDOWN_TIME_SECONDS
self.max_cooldown_time = max_cooldown_time
self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time)
self.disable_cooldowns = disable_cooldowns
self.fallback_on_unsupported_params = fallback_on_unsupported_params
@ -4742,6 +4746,7 @@ class Router:
original_model_id: Final = model_info.get("id")
model_info["id"] = _model_id
model_info["original_model_id"] = original_model_id
model_info["is_clientside_credential"] = True
deployment_pydantic_obj: Final = Deployment(
model_name=model_group,
litellm_params=LiteLLM_Params.model_validate(dynamic_litellm_params),
@ -4805,6 +4810,7 @@ class Router:
model_info = deployment_pydantic_obj.model_info.model_dump()
deployment_litellm_model_name = deployment_pydantic_obj.litellm_params.model
deployment_api_base = deployment_pydantic_obj.litellm_params.api_base
kwargs["is_clientside_credential"] = True
metadata_variable_name: Final = get_router_metadata_variable_name(
function_name=function_name,
@ -9573,6 +9579,13 @@ class Router:
)
return False
if is_client_credential_rate_limit(kwargs, exception_status, exception):
verbose_router_logger.debug(
"Router: Exiting 'deployment_callback_on_failure' without cooldown. "
"Rate limit error (429) was caused by client-supplied / forwarded credentials, not deployment health."
)
return False
exception_headers: Final = litellm.litellm_core_utils.exception_mapping_utils._get_response_headers(
original_exception=exception
)
@ -9596,7 +9609,11 @@ class Router:
if deployment_cooldown is not None and deployment_cooldown >= 0:
_time_to_cooldown = deployment_cooldown
elif header_cooldown is not None and header_cooldown >= 0:
_time_to_cooldown = header_cooldown
_max_cd: Final = getattr(self, "max_cooldown_time", None) or DEFAULT_MAX_COOLDOWN_TIME_SECONDS
if _max_cd is not None and _max_cd >= 0:
_time_to_cooldown = min(header_cooldown, _max_cd)
else:
_time_to_cooldown = header_cooldown
else:
_time_to_cooldown = self.cooldown_time

View file

@ -11,9 +11,10 @@ If given, generate a unique model_id for the deployment.
Ensures cooldowns are applied correctly.
"""
from collections.abc import MutableMapping
from collections.abc import Mapping, MutableMapping
from typing import Final, cast # noqa: TID251 # narrows the untyped request dict for mutation
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.credential_accessor import sends_api_key_secret_reference
from litellm.types.utils import server_owned_wif_litellm_params
@ -84,11 +85,34 @@ def _admin_config_fields_to_clear_on_base_override() -> list[str]:
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE: Final = _admin_config_fields_to_clear_on_base_override()
def is_clientside_credential(request_kwargs: dict) -> bool:
CLIENTSIDE_CREDENTIAL_HEADER_NAMES: Final = frozenset(
{
"x-api-key",
"authorization",
"x-goog-api-key",
"api-key",
"ocp-apim-subscription-key",
}
)
def is_clientside_credential(request_kwargs: Mapping[str, object] | dict) -> bool:
"""
Check if the credential is a clientside credential.
"""
return any(key in request_kwargs for key in clientside_credential_keys)
if not isinstance(request_kwargs, Mapping):
return False
if any(key in request_kwargs for key in clientside_credential_keys):
return True
if request_kwargs.get("has_forwarded_client_auth") is True:
return True
for header_slot in ("headers", "extra_headers", "default_headers"):
headers_val = request_kwargs.get(header_slot)
if isinstance(headers_val, Mapping):
for h in headers_val:
if str(h).lower() in CLIENTSIDE_CREDENTIAL_HEADER_NAMES:
return True
return False
def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> dict:
@ -105,6 +129,13 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di
if key in request_kwargs:
litellm_params[key] = request_kwargs[key]
for header_slot in ("headers", "extra_headers", "default_headers"):
headers_val = request_kwargs.get(header_slot)
if isinstance(headers_val, Mapping):
for h, v in headers_val.items():
if str(h).lower() in CLIENTSIDE_CREDENTIAL_HEADER_NAMES:
litellm_params.setdefault(header_slot, {})[h] = v
# If the caller redirected api_base/base_url to a client-controlled value,
# don't forward the admin's organization / extra_body / region / token /
# vertex / aws fields — those were meant for the original upstream.

View file

@ -807,3 +807,45 @@ def is_caller_scoped_auth_failure(deployment: Deployment | None, exception_statu
)
credential_token_exchange_endpoint: Final = credential_values.get("token_exchange_endpoint")
return isinstance(credential_token_exchange_endpoint, str) and bool(credential_token_exchange_endpoint)
def is_client_credential_rate_limit(
model_call_details: Mapping[str, object],
exception_status: str | int,
exception: Exception | None = None,
) -> bool:
"""A 429 RateLimitError that stems from client-supplied or forwarded credentials
(BYOK) describes the caller's own account quota / subscription, not the health
of the shared deployment. Exits failure callback without cooling down the deployment.
"""
status_code: Final = cast_exception_status_to_int(exception_status)
is_429 = status_code == 429 or (exception is not None and isinstance(exception, litellm.RateLimitError))
if not is_429:
return False
if model_call_details.get("is_clientside_credential") is True:
return True
if model_call_details.get("has_forwarded_client_auth") is True:
return True
litellm_params: Final = model_call_details.get("litellm_params")
if isinstance(litellm_params, Mapping):
if litellm_params.get("is_clientside_credential") is True:
return True
if litellm_params.get("has_forwarded_client_auth") is True:
return True
model_info: Final = litellm_params.get("model_info")
if isinstance(model_info, Mapping):
if model_info.get("is_clientside_credential") is True:
return True
if model_info.get("original_model_id") is not None:
return True
metadata: Final = litellm_params.get("metadata")
if isinstance(metadata, Mapping) and metadata.get("is_clientside_credential") is True:
return True
proxy_req: Final = litellm_params.get("proxy_server_request")
if isinstance(proxy_req, Mapping) and proxy_req.get("credential_fields"):
return True
return False

View file

@ -12,6 +12,7 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_router_logger
from litellm.constants import DEFAULT_MAX_COOLDOWN_TIME_SECONDS
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy
from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference
@ -30,6 +31,8 @@ from litellm.router_utils.cooldown_handlers import (
is_caller_scoped_auth_failure,
is_caller_timeout_408,
is_stream_stall,
is_client_credential_rate_limit,
set_cooldown_deployments,
)
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
@ -117,6 +120,13 @@ def _trigger_cooldown_for_failed_deployment(
)
return
if is_client_credential_rate_limit(model_call_details, exception_status, exception):
verbose_router_logger.debug(
"Not triggering cooldown for fallback deployment: "
"Rate limit error (429) was caused by client-supplied / forwarded credentials, not deployment health."
)
return
# Only Router._set_failed_deployment_id_on_exception()'s server-stamped id is
# trusted here: a metadata-bucket lookup (e.g. "metadata"/"litellm_metadata")
# can't reliably tell a caller-supplied bucket from a router-authored one
@ -152,9 +162,15 @@ def _trigger_cooldown_for_failed_deployment(
_get_retry_after: Final = (
litellm.utils._get_retry_after_from_exception_header # pyright: ignore[reportPrivateUsage] - as router.py
)
header_cooldown: Final = (
raw_header_cooldown: Final = (
_get_retry_after(response_headers=exception_headers) if exception_headers is not None else None
)
_max_cd: Final = getattr(litellm_router, "max_cooldown_time", None) or DEFAULT_MAX_COOLDOWN_TIME_SECONDS
header_cooldown: Final = (
min(raw_header_cooldown, _max_cd)
if raw_header_cooldown is not None and raw_header_cooldown >= 0 and _max_cd is not None and _max_cd >= 0
else raw_header_cooldown
)
time_to_cooldown: Final = (
deployment_cooldown
if deployment_cooldown is not None and deployment_cooldown >= 0

View file

@ -0,0 +1,282 @@
"""Regression tests for Issue #45312:
BYOK rate-limit (429) cooldown isolation and response-header retry-after cooldown capping.
"""
from unittest.mock import patch
import httpx
import pytest
import litellm
from litellm import Router
from litellm.constants import DEFAULT_MAX_COOLDOWN_TIME_SECONDS
from litellm.types.router import AllowedFailsPolicy
@pytest.fixture
def mock_httpx_429():
"""Mock httpx.AsyncClient.send to always return a 429 with a large retry-after."""
async def _mock_send(self, request, **kw):
return httpx.Response(
429,
headers={"retry-after": "86400"},
request=request,
json={"type": "error", "error": {"type": "rate_limit_error", "message": "usage limit"}},
)
with patch.object(httpx.AsyncClient, "send", new=_mock_send):
yield
@pytest.mark.asyncio
async def test_byok_rate_limit_with_extra_headers_does_not_cool_down_shared_deployment(mock_httpx_429):
"""Client providing x-api-key via extra_headers (BYOK) hitting 429 must not cool down the shared deployment."""
router = Router(
num_retries=0,
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
model_list=[
{
"model_name": "claude",
"model_info": {"id": "claude-dep-shared"},
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
}
],
)
# Client 1 makes requests that hit 429 twice (exceeding allowed fails)
for _ in range(2):
with pytest.raises(litellm.RateLimitError):
await router.acompletion(
model="claude",
messages=[{"role": "user", "content": "hi"}],
extra_headers={"x-api-key": "sk-client-tenant-1"},
)
# The shared deployment must remain uncooled
active = await router.cooldown_cache.async_get_active_cooldowns(
model_ids=["claude-dep-shared"], parent_otel_span=None
)
assert len(active) == 0, f"Shared deployment was cooled down: {active}"
@pytest.mark.asyncio
async def test_byok_rate_limit_with_client_api_key_does_not_cool_down_shared_deployment(mock_httpx_429):
"""Client providing dynamic api_key hitting 429 must not cool down the shared deployment."""
router = Router(
num_retries=0,
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
model_list=[
{
"model_name": "claude",
"model_info": {"id": "claude-dep-shared"},
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
}
],
)
for _ in range(2):
with pytest.raises(litellm.RateLimitError):
await router.acompletion(
model="claude",
messages=[{"role": "user", "content": "hi"}],
api_key="sk-client-tenant-1",
)
active = await router.cooldown_cache.async_get_active_cooldowns(
model_ids=["claude-dep-shared"], parent_otel_span=None
)
assert len(active) == 0
@pytest.mark.asyncio
async def test_byok_rate_limit_with_authorization_header_does_not_cool_down_shared_deployment(mock_httpx_429):
"""Client providing OAuth/Bearer Authorization header hitting 429 must not cool down the shared deployment."""
router = Router(
num_retries=0,
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
model_list=[
{
"model_name": "claude",
"model_info": {"id": "claude-dep-shared"},
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
}
],
)
for _ in range(2):
with pytest.raises(litellm.RateLimitError):
await router.acompletion(
model="claude",
messages=[{"role": "user", "content": "hi"}],
extra_headers={"Authorization": "Bearer sk-ant-oauth-token-123"},
)
active = await router.cooldown_cache.async_get_active_cooldowns(
model_ids=["claude-dep-shared"], parent_otel_span=None
)
assert len(active) == 0
@pytest.mark.asyncio
async def test_max_cooldown_time_caps_response_header_retry_after(mock_httpx_429):
"""When max_cooldown_time is configured, a response retry-after header must be capped to that value."""
router = Router(
num_retries=0,
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
max_cooldown_time=120,
model_list=[
{
"model_name": "claude",
"model_info": {"id": "claude-dep-shared"},
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
}
],
)
for _ in range(2):
with pytest.raises(litellm.RateLimitError):
await router.acompletion(
model="claude",
messages=[{"role": "user", "content": "hi"}],
)
active = await router.cooldown_cache.async_get_active_cooldowns(
model_ids=["claude-dep-shared"], parent_otel_span=None
)
assert len(active) == 1
assert active[0][1]["cooldown_time"] == 120
@pytest.mark.asyncio
async def test_default_max_cooldown_time_caps_excessive_retry_after(mock_httpx_429):
"""When max_cooldown_time is not set, a huge retry-after (e.g. 86400s) must be capped to DEFAULT_MAX_COOLDOWN_TIME_SECONDS."""
router = Router(
num_retries=0,
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
model_list=[
{
"model_name": "claude",
"model_info": {"id": "claude-dep-shared"},
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
}
],
)
for _ in range(2):
with pytest.raises(litellm.RateLimitError):
await router.acompletion(
model="claude",
messages=[{"role": "user", "content": "hi"}],
)
active = await router.cooldown_cache.async_get_active_cooldowns(
model_ids=["claude-dep-shared"], parent_otel_span=None
)
assert len(active) == 1
assert active[0][1]["cooldown_time"] == DEFAULT_MAX_COOLDOWN_TIME_SECONDS
def test_is_client_credential_rate_limit_edge_cases():
"""Unit tests for is_client_credential_rate_limit covering all branch conditions."""
from litellm.router_utils.cooldown_handlers import is_client_credential_rate_limit
# Non-429 errors must not be treated as client credential rate limits
assert not is_client_credential_rate_limit({}, exception_status=500)
assert not is_client_credential_rate_limit({}, exception_status="502")
# RateLimitError with no client credentials returns False
rate_err = litellm.RateLimitError(message="rate limited", model="claude", llm_provider="anthropic")
assert not is_client_credential_rate_limit({}, exception_status=429, exception=rate_err)
# Direct model_call_details flags
assert is_client_credential_rate_limit({"is_clientside_credential": True}, exception_status=429)
assert is_client_credential_rate_limit({"has_forwarded_client_auth": True}, exception_status=429)
# Flags inside litellm_params
assert is_client_credential_rate_limit(
{"litellm_params": {"is_clientside_credential": True}}, exception_status=429
)
assert is_client_credential_rate_limit(
{"litellm_params": {"has_forwarded_client_auth": True}}, exception_status=429
)
assert is_client_credential_rate_limit(
{"litellm_params": {"model_info": {"is_clientside_credential": True}}}, exception_status=429
)
assert is_client_credential_rate_limit(
{"litellm_params": {"model_info": {"original_model_id": "dep-original"}}}, exception_status=429
)
assert is_client_credential_rate_limit(
{"litellm_params": {"metadata": {"is_clientside_credential": True}}}, exception_status=429
)
assert is_client_credential_rate_limit(
{"litellm_params": {"proxy_server_request": {"credential_fields": ["api_key"]}}},
exception_status="429",
)
def test_is_clientside_credential_and_dynamic_params():
"""Unit tests for is_clientside_credential and get_dynamic_litellm_params."""
from litellm.router_utils.clientside_credential_handler import (
get_dynamic_litellm_params,
is_clientside_credential,
)
# Non-mapping input
assert not is_clientside_credential(None) # type: ignore[arg-type]
assert not is_clientside_credential("invalid") # type: ignore[arg-type]
# Explicit flag
assert is_clientside_credential({"has_forwarded_client_auth": True})
# Header variations
assert is_clientside_credential({"headers": {"Authorization": "Bearer token"}})
assert is_clientside_credential({"extra_headers": {"X-Goog-Api-Key": "key"}})
assert is_clientside_credential({"default_headers": {"ocp-apim-subscription-key": "sub-key"}})
assert not is_clientside_credential({"headers": {"User-Agent": "litellm-client"}})
# get_dynamic_litellm_params forwards client credential headers
base_params = {"model": "claude"}
updated = get_dynamic_litellm_params(
base_params, {"extra_headers": {"x-api-key": "secret", "unrelated": "val"}}
)
assert updated.get("extra_headers", {}).get("x-api-key") == "secret"
def test_fallback_handler_cooldown_capping_and_bypass():
"""Unit tests for fallback handler retry-after capping and BYOK cooldown bypass."""
from unittest.mock import MagicMock, patch
from litellm.router_utils.fallback_event_handlers import _trigger_cooldown_for_failed_deployment
mock_router = MagicMock()
mock_router.max_cooldown_time = 60
mock_router.get_model_info.return_value = {"litellm_params": {}}
rate_err = litellm.RateLimitError(message="rate limited", model="claude", llm_provider="anthropic")
setattr(rate_err, "failed_deployment_id", "dep-1")
setattr(rate_err, "response", MagicMock(headers={"retry-after": "100000"}))
# BYOK 429 must return early without setting cooldown
_trigger_cooldown_for_failed_deployment(
litellm_router=mock_router,
kwargs={"original_exception": rate_err},
exception=rate_err,
model_call_details={"is_clientside_credential": True},
)
mock_router.set_deployment_cooldown.assert_not_called()
# Shared failure with excessive retry-after should cap to max_cooldown_time
with patch(
"litellm.router_utils.fallback_event_handlers.set_cooldown_deployments"
) as mock_set_cooldown:
_trigger_cooldown_for_failed_deployment(
litellm_router=mock_router,
kwargs={"original_exception": rate_err},
exception=rate_err,
model_call_details={},
)
assert mock_set_cooldown.called
call_kwargs = mock_set_cooldown.call_args.kwargs
assert call_kwargs.get("time_to_cooldown") == 60