diff --git a/litellm/constants.py b/litellm/constants.py index d1daf2ee2d1..2a2030949a7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -151,6 +151,7 @@ DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096) DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3) DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1) DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5) +DEFAULT_MAX_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_MAX_COOLDOWN_TIME_SECONDS", 3600) DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = get_env_float("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", 1.0) DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLING_RETRIES", 5) DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8dc7501bc3b..6ebe81a21cc 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2492,11 +2492,13 @@ async def add_litellm_data_to_request( verbose_proxy_logger.debug("Request Headers: %s", _logging_safe_headers) verbose_proxy_logger.debug("Raw Headers: %s", _raw_headers) - if forward_llm_auth and "x-api-key" in _headers: - data["api_key"] = _headers["x-api-key"] - verbose_proxy_logger.debug( - "Setting client-provided x-api-key as api_key parameter (will override deployment key)" - ) + if forward_llm_auth: + data["has_forwarded_client_auth"] = True + if "x-api-key" in _headers: + data["api_key"] = _headers["x-api-key"] + verbose_proxy_logger.debug( + "Setting client-provided x-api-key as api_key parameter (will override deployment key)" + ) ########################################################## # Init - Proxy Server Request diff --git a/litellm/router.py b/litellm/router.py index 7993b5b921c..b94cf0a0104 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -69,6 +69,7 @@ from litellm.constants import ( DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, DEFAULT_HEALTH_CHECK_INTERVAL, DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, + DEFAULT_MAX_COOLDOWN_TIME_SECONDS, DEFAULT_MAX_LRU_CACHE_SIZE, INTERNAL_CALL_ORIGIN_METADATA_KEY, KUBERNETES_POD_ROUTING_KEY, @@ -240,6 +241,7 @@ from litellm.router_utils.cooldown_handlers import ( is_background_response_poll_not_found, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_client_credential_rate_limit, is_video_lookup_not_found, set_cooldown_deployments, ) @@ -1226,6 +1228,7 @@ class Router: allowed_fails: int | None = None, # Number of times a deployment can failbefore being added to cooldown allowed_fails_policy: AllowedFailsPolicy | None = None, # set custom allowed fails policy cooldown_time: float | None = None, # (seconds) time to cooldown a deployment after failure + max_cooldown_time: float | None = None, # (seconds) maximum cap for response-header cooldowns disable_cooldowns: bool | None = None, fallback_on_unsupported_params: bool = True, routing_strategy: RoutingStrategyName = "simple-shuffle", @@ -1491,7 +1494,8 @@ class Router: self.allowed_fails = allowed_fails else: self.allowed_fails = litellm.allowed_fails - self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS + self.cooldown_time = cooldown_time if cooldown_time is not None else DEFAULT_COOLDOWN_TIME_SECONDS + self.max_cooldown_time = max_cooldown_time self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) self.disable_cooldowns = disable_cooldowns self.fallback_on_unsupported_params = fallback_on_unsupported_params @@ -4742,6 +4746,7 @@ class Router: original_model_id: Final = model_info.get("id") model_info["id"] = _model_id model_info["original_model_id"] = original_model_id + model_info["is_clientside_credential"] = True deployment_pydantic_obj: Final = Deployment( model_name=model_group, litellm_params=LiteLLM_Params.model_validate(dynamic_litellm_params), @@ -4805,6 +4810,7 @@ class Router: model_info = deployment_pydantic_obj.model_info.model_dump() deployment_litellm_model_name = deployment_pydantic_obj.litellm_params.model deployment_api_base = deployment_pydantic_obj.litellm_params.api_base + kwargs["is_clientside_credential"] = True metadata_variable_name: Final = get_router_metadata_variable_name( function_name=function_name, @@ -9573,6 +9579,13 @@ class Router: ) return False + if is_client_credential_rate_limit(kwargs, exception_status, exception): + verbose_router_logger.debug( + "Router: Exiting 'deployment_callback_on_failure' without cooldown. " + "Rate limit error (429) was caused by client-supplied / forwarded credentials, not deployment health." + ) + return False + exception_headers: Final = litellm.litellm_core_utils.exception_mapping_utils._get_response_headers( original_exception=exception ) @@ -9596,7 +9609,11 @@ class Router: if deployment_cooldown is not None and deployment_cooldown >= 0: _time_to_cooldown = deployment_cooldown elif header_cooldown is not None and header_cooldown >= 0: - _time_to_cooldown = header_cooldown + _max_cd: Final = getattr(self, "max_cooldown_time", None) or DEFAULT_MAX_COOLDOWN_TIME_SECONDS + if _max_cd is not None and _max_cd >= 0: + _time_to_cooldown = min(header_cooldown, _max_cd) + else: + _time_to_cooldown = header_cooldown else: _time_to_cooldown = self.cooldown_time diff --git a/litellm/router_utils/clientside_credential_handler.py b/litellm/router_utils/clientside_credential_handler.py index 2bd58405693..a0b30b46909 100644 --- a/litellm/router_utils/clientside_credential_handler.py +++ b/litellm/router_utils/clientside_credential_handler.py @@ -11,9 +11,10 @@ If given, generate a unique model_id for the deployment. Ensures cooldowns are applied correctly. """ -from collections.abc import MutableMapping +from collections.abc import Mapping, MutableMapping from typing import Final, cast # noqa: TID251 # narrows the untyped request dict for mutation + from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.credential_accessor import sends_api_key_secret_reference from litellm.types.utils import server_owned_wif_litellm_params @@ -84,11 +85,34 @@ def _admin_config_fields_to_clear_on_base_override() -> list[str]: _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE: Final = _admin_config_fields_to_clear_on_base_override() -def is_clientside_credential(request_kwargs: dict) -> bool: +CLIENTSIDE_CREDENTIAL_HEADER_NAMES: Final = frozenset( + { + "x-api-key", + "authorization", + "x-goog-api-key", + "api-key", + "ocp-apim-subscription-key", + } +) + + +def is_clientside_credential(request_kwargs: Mapping[str, object] | dict) -> bool: """ Check if the credential is a clientside credential. """ - return any(key in request_kwargs for key in clientside_credential_keys) + if not isinstance(request_kwargs, Mapping): + return False + if any(key in request_kwargs for key in clientside_credential_keys): + return True + if request_kwargs.get("has_forwarded_client_auth") is True: + return True + for header_slot in ("headers", "extra_headers", "default_headers"): + headers_val = request_kwargs.get(header_slot) + if isinstance(headers_val, Mapping): + for h in headers_val: + if str(h).lower() in CLIENTSIDE_CREDENTIAL_HEADER_NAMES: + return True + return False def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> dict: @@ -105,6 +129,13 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di if key in request_kwargs: litellm_params[key] = request_kwargs[key] + for header_slot in ("headers", "extra_headers", "default_headers"): + headers_val = request_kwargs.get(header_slot) + if isinstance(headers_val, Mapping): + for h, v in headers_val.items(): + if str(h).lower() in CLIENTSIDE_CREDENTIAL_HEADER_NAMES: + litellm_params.setdefault(header_slot, {})[h] = v + # If the caller redirected api_base/base_url to a client-controlled value, # don't forward the admin's organization / extra_body / region / token / # vertex / aws fields — those were meant for the original upstream. diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index c362c6899db..cc19c1733b3 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -807,3 +807,45 @@ def is_caller_scoped_auth_failure(deployment: Deployment | None, exception_statu ) credential_token_exchange_endpoint: Final = credential_values.get("token_exchange_endpoint") return isinstance(credential_token_exchange_endpoint, str) and bool(credential_token_exchange_endpoint) + + +def is_client_credential_rate_limit( + model_call_details: Mapping[str, object], + exception_status: str | int, + exception: Exception | None = None, +) -> bool: + """A 429 RateLimitError that stems from client-supplied or forwarded credentials + (BYOK) describes the caller's own account quota / subscription, not the health + of the shared deployment. Exits failure callback without cooling down the deployment. + """ + status_code: Final = cast_exception_status_to_int(exception_status) + is_429 = status_code == 429 or (exception is not None and isinstance(exception, litellm.RateLimitError)) + if not is_429: + return False + + if model_call_details.get("is_clientside_credential") is True: + return True + if model_call_details.get("has_forwarded_client_auth") is True: + return True + + litellm_params: Final = model_call_details.get("litellm_params") + if isinstance(litellm_params, Mapping): + if litellm_params.get("is_clientside_credential") is True: + return True + if litellm_params.get("has_forwarded_client_auth") is True: + return True + model_info: Final = litellm_params.get("model_info") + if isinstance(model_info, Mapping): + if model_info.get("is_clientside_credential") is True: + return True + if model_info.get("original_model_id") is not None: + return True + metadata: Final = litellm_params.get("metadata") + if isinstance(metadata, Mapping) and metadata.get("is_clientside_credential") is True: + return True + proxy_req: Final = litellm_params.get("proxy_server_request") + if isinstance(proxy_req, Mapping) and proxy_req.get("credential_fields"): + return True + + return False + diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 6c93fe6d239..47e04d607eb 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -12,6 +12,7 @@ from pydantic import TypeAdapter import litellm from litellm._logging import verbose_router_logger +from litellm.constants import DEFAULT_MAX_COOLDOWN_TIME_SECONDS from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference @@ -30,6 +31,8 @@ from litellm.router_utils.cooldown_handlers import ( is_caller_scoped_auth_failure, is_caller_timeout_408, is_stream_stall, + is_client_credential_rate_limit, + set_cooldown_deployments, ) from litellm.router_utils.router_callbacks.track_deployment_metrics import ( @@ -117,6 +120,13 @@ def _trigger_cooldown_for_failed_deployment( ) return + if is_client_credential_rate_limit(model_call_details, exception_status, exception): + verbose_router_logger.debug( + "Not triggering cooldown for fallback deployment: " + "Rate limit error (429) was caused by client-supplied / forwarded credentials, not deployment health." + ) + return + # Only Router._set_failed_deployment_id_on_exception()'s server-stamped id is # trusted here: a metadata-bucket lookup (e.g. "metadata"/"litellm_metadata") # can't reliably tell a caller-supplied bucket from a router-authored one @@ -152,9 +162,15 @@ def _trigger_cooldown_for_failed_deployment( _get_retry_after: Final = ( litellm.utils._get_retry_after_from_exception_header # pyright: ignore[reportPrivateUsage] - as router.py ) - header_cooldown: Final = ( + raw_header_cooldown: Final = ( _get_retry_after(response_headers=exception_headers) if exception_headers is not None else None ) + _max_cd: Final = getattr(litellm_router, "max_cooldown_time", None) or DEFAULT_MAX_COOLDOWN_TIME_SECONDS + header_cooldown: Final = ( + min(raw_header_cooldown, _max_cd) + if raw_header_cooldown is not None and raw_header_cooldown >= 0 and _max_cd is not None and _max_cd >= 0 + else raw_header_cooldown + ) time_to_cooldown: Final = ( deployment_cooldown if deployment_cooldown is not None and deployment_cooldown >= 0 diff --git a/tests/unit/test_router/test_byok_cooldown_isolation.py b/tests/unit/test_router/test_byok_cooldown_isolation.py new file mode 100644 index 00000000000..196528ced52 --- /dev/null +++ b/tests/unit/test_router/test_byok_cooldown_isolation.py @@ -0,0 +1,177 @@ +"""Regression tests for Issue #45312: +BYOK rate-limit (429) cooldown isolation and response-header retry-after cooldown capping. +""" + +from unittest.mock import patch + +import httpx +import pytest + +import litellm +from litellm import Router +from litellm.constants import DEFAULT_MAX_COOLDOWN_TIME_SECONDS +from litellm.types.router import AllowedFailsPolicy + + +@pytest.fixture +def mock_httpx_429(): + """Mock httpx.AsyncClient.send to always return a 429 with a large retry-after.""" + + async def _mock_send(self, request, **kw): + return httpx.Response( + 429, + headers={"retry-after": "86400"}, + request=request, + json={"type": "error", "error": {"type": "rate_limit_error", "message": "usage limit"}}, + ) + + with patch.object(httpx.AsyncClient, "send", new=_mock_send): + yield + + +@pytest.mark.asyncio +async def test_byok_rate_limit_with_extra_headers_does_not_cool_down_shared_deployment(mock_httpx_429): + """Client providing x-api-key via extra_headers (BYOK) hitting 429 must not cool down the shared deployment.""" + router = Router( + num_retries=0, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1), + model_list=[ + { + "model_name": "claude", + "model_info": {"id": "claude-dep-shared"}, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"}, + } + ], + ) + + # Client 1 makes requests that hit 429 twice (exceeding allowed fails) + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="claude", + messages=[{"role": "user", "content": "hi"}], + extra_headers={"x-api-key": "sk-client-tenant-1"}, + ) + + # The shared deployment must remain uncooled + active = await router.cooldown_cache.async_get_active_cooldowns( + model_ids=["claude-dep-shared"], parent_otel_span=None + ) + assert len(active) == 0, f"Shared deployment was cooled down: {active}" + + +@pytest.mark.asyncio +async def test_byok_rate_limit_with_client_api_key_does_not_cool_down_shared_deployment(mock_httpx_429): + """Client providing dynamic api_key hitting 429 must not cool down the shared deployment.""" + router = Router( + num_retries=0, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1), + model_list=[ + { + "model_name": "claude", + "model_info": {"id": "claude-dep-shared"}, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"}, + } + ], + ) + + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="claude", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-client-tenant-1", + ) + + active = await router.cooldown_cache.async_get_active_cooldowns( + model_ids=["claude-dep-shared"], parent_otel_span=None + ) + assert len(active) == 0 + + +@pytest.mark.asyncio +async def test_byok_rate_limit_with_authorization_header_does_not_cool_down_shared_deployment(mock_httpx_429): + """Client providing OAuth/Bearer Authorization header hitting 429 must not cool down the shared deployment.""" + router = Router( + num_retries=0, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1), + model_list=[ + { + "model_name": "claude", + "model_info": {"id": "claude-dep-shared"}, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"}, + } + ], + ) + + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="claude", + messages=[{"role": "user", "content": "hi"}], + extra_headers={"Authorization": "Bearer sk-ant-oauth-token-123"}, + ) + + active = await router.cooldown_cache.async_get_active_cooldowns( + model_ids=["claude-dep-shared"], parent_otel_span=None + ) + assert len(active) == 0 + + +@pytest.mark.asyncio +async def test_max_cooldown_time_caps_response_header_retry_after(mock_httpx_429): + """When max_cooldown_time is configured, a response retry-after header must be capped to that value.""" + router = Router( + num_retries=0, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1), + max_cooldown_time=120, + model_list=[ + { + "model_name": "claude", + "model_info": {"id": "claude-dep-shared"}, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"}, + } + ], + ) + + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="claude", + messages=[{"role": "user", "content": "hi"}], + ) + + active = await router.cooldown_cache.async_get_active_cooldowns( + model_ids=["claude-dep-shared"], parent_otel_span=None + ) + assert len(active) == 1 + assert active[0][1]["cooldown_time"] == 120 + + +@pytest.mark.asyncio +async def test_default_max_cooldown_time_caps_excessive_retry_after(mock_httpx_429): + """When max_cooldown_time is not set, a huge retry-after (e.g. 86400s) must be capped to DEFAULT_MAX_COOLDOWN_TIME_SECONDS.""" + router = Router( + num_retries=0, + allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1), + model_list=[ + { + "model_name": "claude", + "model_info": {"id": "claude-dep-shared"}, + "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"}, + } + ], + ) + + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await router.acompletion( + model="claude", + messages=[{"role": "user", "content": "hi"}], + ) + + active = await router.cooldown_cache.async_get_active_cooldowns( + model_ids=["claude-dep-shared"], parent_otel_span=None + ) + assert len(active) == 1 + assert active[0][1]["cooldown_time"] == DEFAULT_MAX_COOLDOWN_TIME_SECONDS