mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-21 10:15:47 +00:00
Merge 9db76e9e29 into dddbaa803d
This commit is contained in:
commit
46b347a071
7 changed files with 402 additions and 11 deletions
|
|
@ -151,6 +151,7 @@ DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096)
|
|||
DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3)
|
||||
DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1)
|
||||
DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5)
|
||||
DEFAULT_MAX_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_MAX_COOLDOWN_TIME_SECONDS", 3600)
|
||||
DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = get_env_float("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", 1.0)
|
||||
DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLING_RETRIES", 5)
|
||||
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
|
||||
|
|
|
|||
|
|
@ -2492,11 +2492,13 @@ async def add_litellm_data_to_request(
|
|||
verbose_proxy_logger.debug("Request Headers: %s", _logging_safe_headers)
|
||||
verbose_proxy_logger.debug("Raw Headers: %s", _raw_headers)
|
||||
|
||||
if forward_llm_auth and "x-api-key" in _headers:
|
||||
data["api_key"] = _headers["x-api-key"]
|
||||
verbose_proxy_logger.debug(
|
||||
"Setting client-provided x-api-key as api_key parameter (will override deployment key)"
|
||||
)
|
||||
if forward_llm_auth:
|
||||
data["has_forwarded_client_auth"] = True
|
||||
if "x-api-key" in _headers:
|
||||
data["api_key"] = _headers["x-api-key"]
|
||||
verbose_proxy_logger.debug(
|
||||
"Setting client-provided x-api-key as api_key parameter (will override deployment key)"
|
||||
)
|
||||
|
||||
##########################################################
|
||||
# Init - Proxy Server Request
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ from litellm.constants import (
|
|||
DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
|
||||
DEFAULT_HEALTH_CHECK_INTERVAL,
|
||||
DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
|
||||
DEFAULT_MAX_COOLDOWN_TIME_SECONDS,
|
||||
DEFAULT_MAX_LRU_CACHE_SIZE,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
KUBERNETES_POD_ROUTING_KEY,
|
||||
|
|
@ -240,6 +241,7 @@ from litellm.router_utils.cooldown_handlers import (
|
|||
is_background_response_poll_not_found,
|
||||
is_caller_scoped_auth_failure,
|
||||
is_caller_timeout_408,
|
||||
is_client_credential_rate_limit,
|
||||
is_video_lookup_not_found,
|
||||
set_cooldown_deployments,
|
||||
)
|
||||
|
|
@ -1226,6 +1228,7 @@ class Router:
|
|||
allowed_fails: int | None = None, # Number of times a deployment can failbefore being added to cooldown
|
||||
allowed_fails_policy: AllowedFailsPolicy | None = None, # set custom allowed fails policy
|
||||
cooldown_time: float | None = None, # (seconds) time to cooldown a deployment after failure
|
||||
max_cooldown_time: float | None = None, # (seconds) maximum cap for response-header cooldowns
|
||||
disable_cooldowns: bool | None = None,
|
||||
fallback_on_unsupported_params: bool = True,
|
||||
routing_strategy: RoutingStrategyName = "simple-shuffle",
|
||||
|
|
@ -1491,7 +1494,8 @@ class Router:
|
|||
self.allowed_fails = allowed_fails
|
||||
else:
|
||||
self.allowed_fails = litellm.allowed_fails
|
||||
self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
|
||||
self.cooldown_time = cooldown_time if cooldown_time is not None else DEFAULT_COOLDOWN_TIME_SECONDS
|
||||
self.max_cooldown_time = max_cooldown_time
|
||||
self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time)
|
||||
self.disable_cooldowns = disable_cooldowns
|
||||
self.fallback_on_unsupported_params = fallback_on_unsupported_params
|
||||
|
|
@ -4742,6 +4746,7 @@ class Router:
|
|||
original_model_id: Final = model_info.get("id")
|
||||
model_info["id"] = _model_id
|
||||
model_info["original_model_id"] = original_model_id
|
||||
model_info["is_clientside_credential"] = True
|
||||
deployment_pydantic_obj: Final = Deployment(
|
||||
model_name=model_group,
|
||||
litellm_params=LiteLLM_Params.model_validate(dynamic_litellm_params),
|
||||
|
|
@ -4805,6 +4810,7 @@ class Router:
|
|||
model_info = deployment_pydantic_obj.model_info.model_dump()
|
||||
deployment_litellm_model_name = deployment_pydantic_obj.litellm_params.model
|
||||
deployment_api_base = deployment_pydantic_obj.litellm_params.api_base
|
||||
kwargs["is_clientside_credential"] = True
|
||||
|
||||
metadata_variable_name: Final = get_router_metadata_variable_name(
|
||||
function_name=function_name,
|
||||
|
|
@ -9573,6 +9579,13 @@ class Router:
|
|||
)
|
||||
return False
|
||||
|
||||
if is_client_credential_rate_limit(kwargs, exception_status, exception):
|
||||
verbose_router_logger.debug(
|
||||
"Router: Exiting 'deployment_callback_on_failure' without cooldown. "
|
||||
"Rate limit error (429) was caused by client-supplied / forwarded credentials, not deployment health."
|
||||
)
|
||||
return False
|
||||
|
||||
exception_headers: Final = litellm.litellm_core_utils.exception_mapping_utils._get_response_headers(
|
||||
original_exception=exception
|
||||
)
|
||||
|
|
@ -9596,7 +9609,11 @@ class Router:
|
|||
if deployment_cooldown is not None and deployment_cooldown >= 0:
|
||||
_time_to_cooldown = deployment_cooldown
|
||||
elif header_cooldown is not None and header_cooldown >= 0:
|
||||
_time_to_cooldown = header_cooldown
|
||||
_max_cd: Final = getattr(self, "max_cooldown_time", None) or DEFAULT_MAX_COOLDOWN_TIME_SECONDS
|
||||
if _max_cd is not None and _max_cd >= 0:
|
||||
_time_to_cooldown = min(header_cooldown, _max_cd)
|
||||
else:
|
||||
_time_to_cooldown = header_cooldown
|
||||
else:
|
||||
_time_to_cooldown = self.cooldown_time
|
||||
|
||||
|
|
|
|||
|
|
@ -11,9 +11,10 @@ If given, generate a unique model_id for the deployment.
|
|||
Ensures cooldowns are applied correctly.
|
||||
"""
|
||||
|
||||
from collections.abc import MutableMapping
|
||||
from collections.abc import Mapping, MutableMapping
|
||||
from typing import Final, cast # noqa: TID251 # narrows the untyped request dict for mutation
|
||||
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.credential_accessor import sends_api_key_secret_reference
|
||||
from litellm.types.utils import server_owned_wif_litellm_params
|
||||
|
|
@ -84,11 +85,34 @@ def _admin_config_fields_to_clear_on_base_override() -> list[str]:
|
|||
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE: Final = _admin_config_fields_to_clear_on_base_override()
|
||||
|
||||
|
||||
def is_clientside_credential(request_kwargs: dict) -> bool:
|
||||
CLIENTSIDE_CREDENTIAL_HEADER_NAMES: Final = frozenset(
|
||||
{
|
||||
"x-api-key",
|
||||
"authorization",
|
||||
"x-goog-api-key",
|
||||
"api-key",
|
||||
"ocp-apim-subscription-key",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def is_clientside_credential(request_kwargs: Mapping[str, object] | dict) -> bool:
|
||||
"""
|
||||
Check if the credential is a clientside credential.
|
||||
"""
|
||||
return any(key in request_kwargs for key in clientside_credential_keys)
|
||||
if not isinstance(request_kwargs, Mapping):
|
||||
return False
|
||||
if any(key in request_kwargs for key in clientside_credential_keys):
|
||||
return True
|
||||
if request_kwargs.get("has_forwarded_client_auth") is True:
|
||||
return True
|
||||
for header_slot in ("headers", "extra_headers", "default_headers"):
|
||||
headers_val = request_kwargs.get(header_slot)
|
||||
if isinstance(headers_val, Mapping):
|
||||
for h in headers_val:
|
||||
if str(h).lower() in CLIENTSIDE_CREDENTIAL_HEADER_NAMES:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> dict:
|
||||
|
|
@ -105,6 +129,13 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di
|
|||
if key in request_kwargs:
|
||||
litellm_params[key] = request_kwargs[key]
|
||||
|
||||
for header_slot in ("headers", "extra_headers", "default_headers"):
|
||||
headers_val = request_kwargs.get(header_slot)
|
||||
if isinstance(headers_val, Mapping):
|
||||
for h, v in headers_val.items():
|
||||
if str(h).lower() in CLIENTSIDE_CREDENTIAL_HEADER_NAMES:
|
||||
litellm_params.setdefault(header_slot, {})[h] = v
|
||||
|
||||
# If the caller redirected api_base/base_url to a client-controlled value,
|
||||
# don't forward the admin's organization / extra_body / region / token /
|
||||
# vertex / aws fields — those were meant for the original upstream.
|
||||
|
|
|
|||
|
|
@ -807,3 +807,45 @@ def is_caller_scoped_auth_failure(deployment: Deployment | None, exception_statu
|
|||
)
|
||||
credential_token_exchange_endpoint: Final = credential_values.get("token_exchange_endpoint")
|
||||
return isinstance(credential_token_exchange_endpoint, str) and bool(credential_token_exchange_endpoint)
|
||||
|
||||
|
||||
def is_client_credential_rate_limit(
|
||||
model_call_details: Mapping[str, object],
|
||||
exception_status: str | int,
|
||||
exception: Exception | None = None,
|
||||
) -> bool:
|
||||
"""A 429 RateLimitError that stems from client-supplied or forwarded credentials
|
||||
(BYOK) describes the caller's own account quota / subscription, not the health
|
||||
of the shared deployment. Exits failure callback without cooling down the deployment.
|
||||
"""
|
||||
status_code: Final = cast_exception_status_to_int(exception_status)
|
||||
is_429 = status_code == 429 or (exception is not None and isinstance(exception, litellm.RateLimitError))
|
||||
if not is_429:
|
||||
return False
|
||||
|
||||
if model_call_details.get("is_clientside_credential") is True:
|
||||
return True
|
||||
if model_call_details.get("has_forwarded_client_auth") is True:
|
||||
return True
|
||||
|
||||
litellm_params: Final = model_call_details.get("litellm_params")
|
||||
if isinstance(litellm_params, Mapping):
|
||||
if litellm_params.get("is_clientside_credential") is True:
|
||||
return True
|
||||
if litellm_params.get("has_forwarded_client_auth") is True:
|
||||
return True
|
||||
model_info: Final = litellm_params.get("model_info")
|
||||
if isinstance(model_info, Mapping):
|
||||
if model_info.get("is_clientside_credential") is True:
|
||||
return True
|
||||
if model_info.get("original_model_id") is not None:
|
||||
return True
|
||||
metadata: Final = litellm_params.get("metadata")
|
||||
if isinstance(metadata, Mapping) and metadata.get("is_clientside_credential") is True:
|
||||
return True
|
||||
proxy_req: Final = litellm_params.get("proxy_server_request")
|
||||
if isinstance(proxy_req, Mapping) and proxy_req.get("credential_fields"):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.constants import DEFAULT_MAX_COOLDOWN_TIME_SECONDS
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy
|
||||
from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference
|
||||
|
|
@ -30,6 +31,8 @@ from litellm.router_utils.cooldown_handlers import (
|
|||
is_caller_scoped_auth_failure,
|
||||
is_caller_timeout_408,
|
||||
is_stream_stall,
|
||||
is_client_credential_rate_limit,
|
||||
|
||||
set_cooldown_deployments,
|
||||
)
|
||||
from litellm.router_utils.router_callbacks.track_deployment_metrics import (
|
||||
|
|
@ -117,6 +120,13 @@ def _trigger_cooldown_for_failed_deployment(
|
|||
)
|
||||
return
|
||||
|
||||
if is_client_credential_rate_limit(model_call_details, exception_status, exception):
|
||||
verbose_router_logger.debug(
|
||||
"Not triggering cooldown for fallback deployment: "
|
||||
"Rate limit error (429) was caused by client-supplied / forwarded credentials, not deployment health."
|
||||
)
|
||||
return
|
||||
|
||||
# Only Router._set_failed_deployment_id_on_exception()'s server-stamped id is
|
||||
# trusted here: a metadata-bucket lookup (e.g. "metadata"/"litellm_metadata")
|
||||
# can't reliably tell a caller-supplied bucket from a router-authored one
|
||||
|
|
@ -152,9 +162,15 @@ def _trigger_cooldown_for_failed_deployment(
|
|||
_get_retry_after: Final = (
|
||||
litellm.utils._get_retry_after_from_exception_header # pyright: ignore[reportPrivateUsage] - as router.py
|
||||
)
|
||||
header_cooldown: Final = (
|
||||
raw_header_cooldown: Final = (
|
||||
_get_retry_after(response_headers=exception_headers) if exception_headers is not None else None
|
||||
)
|
||||
_max_cd: Final = getattr(litellm_router, "max_cooldown_time", None) or DEFAULT_MAX_COOLDOWN_TIME_SECONDS
|
||||
header_cooldown: Final = (
|
||||
min(raw_header_cooldown, _max_cd)
|
||||
if raw_header_cooldown is not None and raw_header_cooldown >= 0 and _max_cd is not None and _max_cd >= 0
|
||||
else raw_header_cooldown
|
||||
)
|
||||
time_to_cooldown: Final = (
|
||||
deployment_cooldown
|
||||
if deployment_cooldown is not None and deployment_cooldown >= 0
|
||||
|
|
|
|||
282
tests/unit/test_router/test_byok_cooldown_isolation.py
Normal file
282
tests/unit/test_router/test_byok_cooldown_isolation.py
Normal file
|
|
@ -0,0 +1,282 @@
|
|||
"""Regression tests for Issue #45312:
|
||||
BYOK rate-limit (429) cooldown isolation and response-header retry-after cooldown capping.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.constants import DEFAULT_MAX_COOLDOWN_TIME_SECONDS
|
||||
from litellm.types.router import AllowedFailsPolicy
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_httpx_429():
|
||||
"""Mock httpx.AsyncClient.send to always return a 429 with a large retry-after."""
|
||||
|
||||
async def _mock_send(self, request, **kw):
|
||||
return httpx.Response(
|
||||
429,
|
||||
headers={"retry-after": "86400"},
|
||||
request=request,
|
||||
json={"type": "error", "error": {"type": "rate_limit_error", "message": "usage limit"}},
|
||||
)
|
||||
|
||||
with patch.object(httpx.AsyncClient, "send", new=_mock_send):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_byok_rate_limit_with_extra_headers_does_not_cool_down_shared_deployment(mock_httpx_429):
|
||||
"""Client providing x-api-key via extra_headers (BYOK) hitting 429 must not cool down the shared deployment."""
|
||||
router = Router(
|
||||
num_retries=0,
|
||||
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude",
|
||||
"model_info": {"id": "claude-dep-shared"},
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
# Client 1 makes requests that hit 429 twice (exceeding allowed fails)
|
||||
for _ in range(2):
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await router.acompletion(
|
||||
model="claude",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
extra_headers={"x-api-key": "sk-client-tenant-1"},
|
||||
)
|
||||
|
||||
# The shared deployment must remain uncooled
|
||||
active = await router.cooldown_cache.async_get_active_cooldowns(
|
||||
model_ids=["claude-dep-shared"], parent_otel_span=None
|
||||
)
|
||||
assert len(active) == 0, f"Shared deployment was cooled down: {active}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_byok_rate_limit_with_client_api_key_does_not_cool_down_shared_deployment(mock_httpx_429):
|
||||
"""Client providing dynamic api_key hitting 429 must not cool down the shared deployment."""
|
||||
router = Router(
|
||||
num_retries=0,
|
||||
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude",
|
||||
"model_info": {"id": "claude-dep-shared"},
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
for _ in range(2):
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await router.acompletion(
|
||||
model="claude",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="sk-client-tenant-1",
|
||||
)
|
||||
|
||||
active = await router.cooldown_cache.async_get_active_cooldowns(
|
||||
model_ids=["claude-dep-shared"], parent_otel_span=None
|
||||
)
|
||||
assert len(active) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_byok_rate_limit_with_authorization_header_does_not_cool_down_shared_deployment(mock_httpx_429):
|
||||
"""Client providing OAuth/Bearer Authorization header hitting 429 must not cool down the shared deployment."""
|
||||
router = Router(
|
||||
num_retries=0,
|
||||
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude",
|
||||
"model_info": {"id": "claude-dep-shared"},
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
for _ in range(2):
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await router.acompletion(
|
||||
model="claude",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
extra_headers={"Authorization": "Bearer sk-ant-oauth-token-123"},
|
||||
)
|
||||
|
||||
active = await router.cooldown_cache.async_get_active_cooldowns(
|
||||
model_ids=["claude-dep-shared"], parent_otel_span=None
|
||||
)
|
||||
assert len(active) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_cooldown_time_caps_response_header_retry_after(mock_httpx_429):
|
||||
"""When max_cooldown_time is configured, a response retry-after header must be capped to that value."""
|
||||
router = Router(
|
||||
num_retries=0,
|
||||
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
|
||||
max_cooldown_time=120,
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude",
|
||||
"model_info": {"id": "claude-dep-shared"},
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
for _ in range(2):
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await router.acompletion(
|
||||
model="claude",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
active = await router.cooldown_cache.async_get_active_cooldowns(
|
||||
model_ids=["claude-dep-shared"], parent_otel_span=None
|
||||
)
|
||||
assert len(active) == 1
|
||||
assert active[0][1]["cooldown_time"] == 120
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_max_cooldown_time_caps_excessive_retry_after(mock_httpx_429):
|
||||
"""When max_cooldown_time is not set, a huge retry-after (e.g. 86400s) must be capped to DEFAULT_MAX_COOLDOWN_TIME_SECONDS."""
|
||||
router = Router(
|
||||
num_retries=0,
|
||||
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=1),
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "claude",
|
||||
"model_info": {"id": "claude-dep-shared"},
|
||||
"litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-ant-deployment-key"},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
for _ in range(2):
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await router.acompletion(
|
||||
model="claude",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
active = await router.cooldown_cache.async_get_active_cooldowns(
|
||||
model_ids=["claude-dep-shared"], parent_otel_span=None
|
||||
)
|
||||
assert len(active) == 1
|
||||
assert active[0][1]["cooldown_time"] == DEFAULT_MAX_COOLDOWN_TIME_SECONDS
|
||||
|
||||
|
||||
def test_is_client_credential_rate_limit_edge_cases():
|
||||
"""Unit tests for is_client_credential_rate_limit covering all branch conditions."""
|
||||
from litellm.router_utils.cooldown_handlers import is_client_credential_rate_limit
|
||||
|
||||
# Non-429 errors must not be treated as client credential rate limits
|
||||
assert not is_client_credential_rate_limit({}, exception_status=500)
|
||||
assert not is_client_credential_rate_limit({}, exception_status="502")
|
||||
|
||||
# RateLimitError with no client credentials returns False
|
||||
rate_err = litellm.RateLimitError(message="rate limited", model="claude", llm_provider="anthropic")
|
||||
assert not is_client_credential_rate_limit({}, exception_status=429, exception=rate_err)
|
||||
|
||||
# Direct model_call_details flags
|
||||
assert is_client_credential_rate_limit({"is_clientside_credential": True}, exception_status=429)
|
||||
assert is_client_credential_rate_limit({"has_forwarded_client_auth": True}, exception_status=429)
|
||||
|
||||
# Flags inside litellm_params
|
||||
assert is_client_credential_rate_limit(
|
||||
{"litellm_params": {"is_clientside_credential": True}}, exception_status=429
|
||||
)
|
||||
assert is_client_credential_rate_limit(
|
||||
{"litellm_params": {"has_forwarded_client_auth": True}}, exception_status=429
|
||||
)
|
||||
assert is_client_credential_rate_limit(
|
||||
{"litellm_params": {"model_info": {"is_clientside_credential": True}}}, exception_status=429
|
||||
)
|
||||
assert is_client_credential_rate_limit(
|
||||
{"litellm_params": {"model_info": {"original_model_id": "dep-original"}}}, exception_status=429
|
||||
)
|
||||
assert is_client_credential_rate_limit(
|
||||
{"litellm_params": {"metadata": {"is_clientside_credential": True}}}, exception_status=429
|
||||
)
|
||||
assert is_client_credential_rate_limit(
|
||||
{"litellm_params": {"proxy_server_request": {"credential_fields": ["api_key"]}}},
|
||||
exception_status="429",
|
||||
)
|
||||
|
||||
|
||||
def test_is_clientside_credential_and_dynamic_params():
|
||||
"""Unit tests for is_clientside_credential and get_dynamic_litellm_params."""
|
||||
from litellm.router_utils.clientside_credential_handler import (
|
||||
get_dynamic_litellm_params,
|
||||
is_clientside_credential,
|
||||
)
|
||||
|
||||
# Non-mapping input
|
||||
assert not is_clientside_credential(None) # type: ignore[arg-type]
|
||||
assert not is_clientside_credential("invalid") # type: ignore[arg-type]
|
||||
|
||||
# Explicit flag
|
||||
assert is_clientside_credential({"has_forwarded_client_auth": True})
|
||||
|
||||
# Header variations
|
||||
assert is_clientside_credential({"headers": {"Authorization": "Bearer token"}})
|
||||
assert is_clientside_credential({"extra_headers": {"X-Goog-Api-Key": "key"}})
|
||||
assert is_clientside_credential({"default_headers": {"ocp-apim-subscription-key": "sub-key"}})
|
||||
assert not is_clientside_credential({"headers": {"User-Agent": "litellm-client"}})
|
||||
|
||||
# get_dynamic_litellm_params forwards client credential headers
|
||||
base_params = {"model": "claude"}
|
||||
updated = get_dynamic_litellm_params(
|
||||
base_params, {"extra_headers": {"x-api-key": "secret", "unrelated": "val"}}
|
||||
)
|
||||
assert updated.get("extra_headers", {}).get("x-api-key") == "secret"
|
||||
|
||||
|
||||
def test_fallback_handler_cooldown_capping_and_bypass():
|
||||
"""Unit tests for fallback handler retry-after capping and BYOK cooldown bypass."""
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from litellm.router_utils.fallback_event_handlers import _trigger_cooldown_for_failed_deployment
|
||||
|
||||
mock_router = MagicMock()
|
||||
mock_router.max_cooldown_time = 60
|
||||
mock_router.get_model_info.return_value = {"litellm_params": {}}
|
||||
|
||||
rate_err = litellm.RateLimitError(message="rate limited", model="claude", llm_provider="anthropic")
|
||||
setattr(rate_err, "failed_deployment_id", "dep-1")
|
||||
setattr(rate_err, "response", MagicMock(headers={"retry-after": "100000"}))
|
||||
|
||||
# BYOK 429 must return early without setting cooldown
|
||||
_trigger_cooldown_for_failed_deployment(
|
||||
litellm_router=mock_router,
|
||||
kwargs={"original_exception": rate_err},
|
||||
exception=rate_err,
|
||||
model_call_details={"is_clientside_credential": True},
|
||||
)
|
||||
mock_router.set_deployment_cooldown.assert_not_called()
|
||||
|
||||
# Shared failure with excessive retry-after should cap to max_cooldown_time
|
||||
with patch(
|
||||
"litellm.router_utils.fallback_event_handlers.set_cooldown_deployments"
|
||||
) as mock_set_cooldown:
|
||||
_trigger_cooldown_for_failed_deployment(
|
||||
litellm_router=mock_router,
|
||||
kwargs={"original_exception": rate_err},
|
||||
exception=rate_err,
|
||||
model_call_details={},
|
||||
)
|
||||
assert mock_set_cooldown.called
|
||||
call_kwargs = mock_set_cooldown.call_args.kwargs
|
||||
assert call_kwargs.get("time_to_cooldown") == 60
|
||||
|
||||
Loading…
Add table
Reference in a new issue