Merge pull request #13529 from BerriAI/litellm_dev_08_11_2025_p1

[Fix] Cooldowns - don't return raw Azure Exceptions to client
This commit is contained in:
Krish Dholakia 2025-08-18 18:54:19 -07:00 • committed by GitHub
commit 3b52545db3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 434 additions and 13 deletions

View file

@ -33,7 +33,12 @@ class SensitiveDataMasker:
value_str = str(value)
masked_length = len(value_str) - (self.visible_prefix + self.visible_suffix)
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
# Handle the case where visible_suffix is 0 to avoid showing the entire string
if self.visible_suffix == 0:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}"
else:
return f"{value_str[:self.visible_prefix]}{self.mask_char * masked_length}{value_str[-self.visible_suffix:]}"
def is_sensitive_key(self, key: str) -> bool:
key_lower = str(key).lower()

View file

@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, Union
from litellm import verbose_logger
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -29,6 +30,12 @@ class CooldownCache:
self.cache = cache
self.default_cooldown_time = default_cooldown_time
self.in_memory_cache = InMemoryCache()
# Initialize the masker with custom settings for exception strings
self.exception_masker = SensitiveDataMasker(
visible_prefix=50, # Show first 50 characters
visible_suffix=0, # Show last 0 characters
mask_char="*", # Use * for masking
)
def _common_add_cooldown_logic(
self, model_id: str, original_exception, exception_status, cooldown_time: float
@ -39,7 +46,9 @@ class CooldownCache:
# Store the cooldown information for the deployment separately
cooldown_data = CooldownCacheValue(
exception_received=str(original_exception),
exception_received=self.exception_masker._mask_value(
str(original_exception)
),
status_code=str(exception_status),
timestamp=current_time,
cooldown_time=cooldown_time,

View file

@ -118,16 +118,16 @@ def _should_run_cooldown_logic(
"Should Not Run Cooldown Logic: deployment id is none or model group can't be found."
)
return False
#########################################################
# If time_to_cooldown is 0 or 0.0000000, don't run cooldown logic
#########################################################
if time_to_cooldown is not None and math.isclose(
a=time_to_cooldown,
b=0.0,
abs_tol=1e-9
a=time_to_cooldown, b=0.0, abs_tol=1e-9
):
verbose_router_logger.debug("Should Not Run Cooldown Logic: time_to_cooldown is effectively 0")
verbose_router_logger.debug(
"Should Not Run Cooldown Logic: time_to_cooldown is effectively 0"
)
return False
if litellm_router_instance.disable_cooldowns:
@ -275,8 +275,8 @@ def _set_cooldown_deployments(
if (
_should_run_cooldown_logic(
litellm_router_instance=litellm_router_instance,
deployment=deployment,
exception_status=exception_status,
deployment=deployment,
exception_status=exception_status,
original_exception=original_exception,
time_to_cooldown=time_to_cooldown,
)
@ -290,9 +290,9 @@ def _set_cooldown_deployments(
verbose_router_logger.debug(f"Attempting to add {deployment} to cooldown list")
if _should_cooldown_deployment(
litellm_router_instance=litellm_router_instance,
deployment=deployment,
exception_status=exception_status,
litellm_router_instance=litellm_router_instance,
deployment=deployment,
exception_status=exception_status,
original_exception=original_exception,
):
litellm_router_instance.cooldown_cache.add_deployment_to_cooldown(

View file

@ -82,9 +82,14 @@ async def async_raise_no_deployment_exception(
litellm_router_instance=litellm_router_instance,
parent_otel_span=parent_otel_span,
)
verbose_router_logger.info(
f"No deployment found for model: {model}, cooldown_list with debug info: {_cooldown_list}"
)
cooldown_list_ids = [cooldown_model[0] for cooldown_model in (_cooldown_list or [])]
return RouterRateLimitError(
model=model,
cooldown_time=_cooldown_time,
enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks,
cooldown_list=_cooldown_list,
cooldown_list=cooldown_list_ids,
)

View file

@ -1,6 +1,7 @@
import sys, os, time
import traceback, asyncio
import pytest
from typing import List
sys.path.insert(
0, os.path.abspath("../..")
@ -111,3 +112,147 @@ async def test_send_llm_exception_alert_when_proxy_server_request_in_kwargs():
# Assert that no exception was raised and the function completed successfully
mock_router.slack_alerting_logger.send_alert.assert_not_called()
@pytest.mark.asyncio
async def test_async_raise_no_deployment_exception():
"""
Test that async_raise_no_deployment_exception returns a RouterRateLimitError
with cooldown_list containing just IDs (not tuples with debug info).
"""
from litellm.router_utils.handle_error import async_raise_no_deployment_exception
from litellm.types.router import RouterRateLimitError
from unittest.mock import patch
# Create a mock LitellmRouter instance
mock_router = MagicMock()
mock_router.get_model_ids.return_value = ["deployment-1", "deployment-2"]
mock_router.cooldown_cache.get_min_cooldown.return_value = 30.0
mock_router.enable_pre_call_checks = True
# Mock the _async_get_cooldown_deployments_with_debug_info function
# It should return a list of tuples where each tuple contains (model_id, debug_info)
mock_cooldown_list = [
("deployment-1", {"error": "rate_limit", "time": "2024-01-01"}),
("deployment-2", {"error": "server_error", "time": "2024-01-01"}),
("deployment-3", {"error": "timeout", "time": "2024-01-01"}),
]
with patch(
"litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info",
return_value=mock_cooldown_list,
):
# Call the function
result = await async_raise_no_deployment_exception(
litellm_router_instance=mock_router,
model="gpt-3.5-turbo",
parent_otel_span=None,
)
# Assert that the function returns a RouterRateLimitError
assert isinstance(result, RouterRateLimitError)
# Assert that the error has the correct properties
assert result.model == "gpt-3.5-turbo"
assert result.cooldown_time == 30.0
assert result.enable_pre_call_checks is True
# Assert that cooldown_list contains only IDs (extracted from tuples)
expected_cooldown_list = ["deployment-1", "deployment-2", "deployment-3"]
assert result.cooldown_list == expected_cooldown_list
# Verify that cooldown_list contains only strings (IDs), not tuples
for item in result.cooldown_list:
assert isinstance(item, str), f"Expected string ID, got {type(item)}: {item}"
# Verify mock calls
mock_router.get_model_ids.assert_called_once_with(model_name="gpt-3.5-turbo")
mock_router.cooldown_cache.get_min_cooldown.assert_called_once_with(
model_ids=["deployment-1", "deployment-2"], parent_otel_span=None
)
@pytest.mark.asyncio
async def test_async_raise_no_deployment_exception_empty_cooldown_list():
"""
Test that async_raise_no_deployment_exception handles empty cooldown list correctly.
"""
from litellm.router_utils.handle_error import async_raise_no_deployment_exception
from litellm.types.router import RouterRateLimitError
from unittest.mock import patch
# Create a mock LitellmRouter instance
mock_router = MagicMock()
mock_router.get_model_ids.return_value = ["deployment-1", "deployment-2"]
mock_router.cooldown_cache.get_min_cooldown.return_value = 15.0
mock_router.enable_pre_call_checks = False
# Mock empty cooldown list
mock_cooldown_list: List = []
with patch(
"litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info",
return_value=mock_cooldown_list,
):
# Call the function
result = await async_raise_no_deployment_exception(
litellm_router_instance=mock_router,
model="claude-3-sonnet",
parent_otel_span=None,
)
# Assert that the function returns a RouterRateLimitError
assert isinstance(result, RouterRateLimitError)
# Assert that the error has the correct properties
assert result.model == "claude-3-sonnet"
assert result.cooldown_time == 15.0
assert result.enable_pre_call_checks is False
# Assert that cooldown_list is an empty list when no cooldowns exist
assert result.cooldown_list == []
assert isinstance(result.cooldown_list, list)
@pytest.mark.asyncio
async def test_async_raise_no_deployment_exception_none_cooldown_list():
"""
Test that async_raise_no_deployment_exception handles None cooldown list correctly.
Note: In practice, _async_get_cooldown_deployments_with_debug_info should never return None
based on the implementation, but this tests defensive programming.
"""
from litellm.router_utils.handle_error import async_raise_no_deployment_exception
from litellm.types.router import RouterRateLimitError
from unittest.mock import patch
# Create a mock LitellmRouter instance
mock_router = MagicMock()
mock_router.get_model_ids.return_value = []
mock_router.cooldown_cache.get_min_cooldown.return_value = 45.0
mock_router.enable_pre_call_checks = True
# Mock None cooldown list (though this shouldn't happen in practice)
mock_cooldown_list = None
with patch(
"litellm.router_utils.handle_error._async_get_cooldown_deployments_with_debug_info",
return_value=mock_cooldown_list,
):
# After the defensive fix, this should handle None gracefully and return empty list
result = await async_raise_no_deployment_exception(
litellm_router_instance=mock_router,
model="gpt-4",
parent_otel_span=None,
)
# Assert that the function returns a RouterRateLimitError
assert isinstance(result, RouterRateLimitError)
# Assert that the error has the correct properties
assert result.model == "gpt-4"
assert result.cooldown_time == 45.0
assert result.enable_pre_call_checks is True
# Assert that cooldown_list is an empty list when cooldown_list is None
assert result.cooldown_list == []
assert isinstance(result.cooldown_list, list)

View file

@ -0,0 +1,257 @@
"""
Unit tests for CooldownCache exception masking functionality
"""
import os
import sys
from unittest.mock import MagicMock
import pytest
# Add the parent directory to the system path
sys.path.insert(0, os.path.abspath("../../.."))
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.router_utils.cooldown_cache import CooldownCache, CooldownCacheValue
class TestCooldownCacheExceptionMasking:
"""Test suite for CooldownCache exception masking functionality"""
@pytest.fixture
def cooldown_cache(self):
"""Create a CooldownCache instance for testing"""
mock_dual_cache = MagicMock(spec=DualCache)
return CooldownCache(cache=mock_dual_cache, default_cooldown_time=60.0)
def test_exception_masker_initialization(self, cooldown_cache):
"""Test that the exception masker is properly initialized"""
assert isinstance(cooldown_cache.exception_masker, SensitiveDataMasker)
assert cooldown_cache.exception_masker.visible_prefix == 50
assert cooldown_cache.exception_masker.visible_suffix == 0
assert cooldown_cache.exception_masker.mask_char == "*"
def test_short_exception_string_not_masked(self, cooldown_cache):
"""Test that short exception strings are not masked"""
short_exception = "Short error"
model_id = "test-model"
exception_status = 500
cooldown_time = 30.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(short_exception),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
# Short exception should not be masked
assert cooldown_data["exception_received"] == short_exception
assert cooldown_key == f"deployment:{model_id}:cooldown"
def test_long_exception_string_masked(self, cooldown_cache):
"""Test that long exception strings are properly masked"""
# Create a long exception string that simulates prompt leakage
long_exception = (
"litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occurred - "
"No deployments available for selected model, Try again in 5 seconds. "
"Passed model=anthropic_claude_sonnet_4_v1_0. pre-call-checks=False, "
"cooldown_list=[('deepseek_r1-eastus', {'exception_received': "
"'litellm.RateLimitError: RateLimitError: Azure_aiException - "
'{"error":{"code":"Invalid input","status":422,"message":"invalid input error",'
'"details":[{"type":"model_attributes_type","loc":["body"],'
'"msg":"Tell me a story about a dragon and a princess in a magical kingdom '
"where the dragon is actually protecting the princess from an evil wizard "
'who wants to steal her magical powers and use them to conquer the world"}]}'
)
model_id = "test-model"
exception_status = 429
cooldown_time = 60.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(long_exception),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
masked_exception = cooldown_data["exception_received"]
# Should start with first 50 characters
assert masked_exception.startswith(long_exception[:50])
# Should contain masking characters
assert "*" in masked_exception
# Should be same length (prefix + asterisks)
assert len(masked_exception) == len(long_exception)
# Should not contain the sensitive prompt content
assert "Tell me a story about a dragon" not in masked_exception
assert "magical kingdom" not in masked_exception
# Should preserve the error type information at the beginning (first 50 chars)
assert masked_exception.startswith(
"litellm.proxy.proxy_server._handle_llm_api_excepti"
)
def test_exception_with_api_keys_masked(self, cooldown_cache):
"""Test that API keys in exceptions are properly masked"""
exception_with_key = (
"Authentication failed with api_key=sk-1234567890abcdefghijklmnopqrstuvwxyz "
"and token=bearer_token_123456789 for model gpt-4"
)
model_id = "test-model"
exception_status = 401
cooldown_time = 30.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(exception_with_key),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
masked_exception = cooldown_data["exception_received"]
# Should mask the sensitive content while preserving structure
assert masked_exception.startswith(
"Authentication failed with api_key=sk-12345678"
)
assert "*" in masked_exception
assert len(masked_exception) == len(exception_with_key)
def test_cooldown_data_structure(self, cooldown_cache):
"""Test that the cooldown data structure is correctly formed"""
exception_msg = "Test exception for structure validation"
model_id = "test-model"
exception_status = 500
cooldown_time = 45.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(exception_msg),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
# Verify cooldown data structure
assert isinstance(cooldown_data, dict)
assert "exception_received" in cooldown_data
assert "status_code" in cooldown_data
assert "timestamp" in cooldown_data
assert "cooldown_time" in cooldown_data
# Verify data types
assert isinstance(cooldown_data["exception_received"], str)
assert isinstance(cooldown_data["status_code"], str)
assert isinstance(cooldown_data["timestamp"], float)
assert isinstance(cooldown_data["cooldown_time"], float)
# Verify values
assert cooldown_data["status_code"] == str(exception_status)
assert cooldown_data["cooldown_time"] == cooldown_time
assert cooldown_data["exception_received"] == exception_msg
def test_exception_object_conversion(self, cooldown_cache):
"""Test that different exception types are properly converted to strings"""
# Test with different exception types
exceptions = [
ValueError("Invalid value provided"),
KeyError("Missing required key"),
RuntimeError("Runtime error occurred"),
Exception("Generic exception"),
]
for exc in exceptions:
model_id = f"test-model-{exc.__class__.__name__}"
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=exc,
exception_status=500,
cooldown_time=30.0,
)
# Should successfully convert exception to string
assert isinstance(cooldown_data["exception_received"], str)
assert (
str(exc) == cooldown_data["exception_received"]
) # Short exceptions not masked
def test_masking_preserves_error_debugging_info(self, cooldown_cache):
"""Test that masking preserves essential debugging information"""
debugging_exception = (
"RateLimitError: Rate limit exceeded for model gpt-4. "
"Current usage: 1000 tokens/minute. Limit: 500 tokens/minute. "
"Request details: model=gpt-4, user_id=user123, "
"prompt='Write a comprehensive analysis of the economic implications "
"of artificial intelligence adoption in the healthcare sector, including "
"potential cost savings, job displacement, and regulatory challenges'"
)
model_id = "gpt-4-deployment"
exception_status = 429
cooldown_time = 120.0
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=Exception(debugging_exception),
exception_status=exception_status,
cooldown_time=cooldown_time,
)
masked_exception = cooldown_data["exception_received"]
# Should preserve error type and initial debugging info (first 50 chars)
assert masked_exception.startswith(
"RateLimitError: Rate limit exceeded for model gpt-"
)
# Should mask the prompt content
assert "Write a comprehensive analysis" not in masked_exception
assert "healthcare sector" not in masked_exception
# Should contain masking indicator
assert "*" in masked_exception
def test_error_handling_in_common_add_cooldown_logic(self, cooldown_cache):
"""Test error handling in the _common_add_cooldown_logic method"""
# This test ensures that edge cases are properly handled
model_id = "test-model"
# Test with None exception (edge case) - should be handled gracefully
cooldown_key, cooldown_data = cooldown_cache._common_add_cooldown_logic(
model_id=model_id,
original_exception=None,
exception_status=500,
cooldown_time=30.0,
)
# Should handle None by converting to string
assert cooldown_data["exception_received"] == "None"
assert cooldown_key == f"deployment:{model_id}:cooldown"
def test_custom_masker_settings(self):
"""Test that custom masker settings work correctly"""
mock_dual_cache = MagicMock(spec=DualCache)
# Create cooldown cache and verify default settings
cache = CooldownCache(cache=mock_dual_cache, default_cooldown_time=60.0)
# Test that we can access and verify the masker configuration
assert cache.exception_masker.visible_prefix == 50
assert cache.exception_masker.visible_suffix == 0
assert cache.exception_masker.mask_char == "*"
# Test masking behavior with these settings
long_string = "A" * 100 # 100 character string
masked = cache.exception_masker._mask_value(long_string)
# Should show first 50 characters, then all asterisks
expected = "A" * 50 + "*" * 50
assert masked == expected