diff --git a/litellm/router.py b/litellm/router.py index feaf69a44ae..864cf950b6a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -155,6 +155,7 @@ from litellm.router_utils.router_callbacks.track_deployment_metrics import ( increment_deployment_successes_for_current_minute, ) from litellm.scheduler import FlowItem, Scheduler +from litellm.types.caching import LiteLLMCacheType from litellm.types.llms.openai import ( AllMessageValues, FileTypes, @@ -345,6 +346,17 @@ def _replay_live_router_model_cost() -> None: set_live_deployment_replay(_replay_live_router_model_cost) +def _resolve_cache_type(requested_type: object) -> LiteLLMCacheType: + """Resolve the `type` entry of `cache_kwargs` when no redis connection params are given.""" + if requested_type is None: + return LiteLLMCacheType.LOCAL + if isinstance(requested_type, LiteLLMCacheType): + return requested_type + if isinstance(requested_type, str): + return LiteLLMCacheType(requested_type) + raise ValueError(f"cache_kwargs['type'] must be a string cache type, got {type(requested_type).__name__}") + + # Kwargs that log_retry must not copy into a retry breadcrumb. The breadcrumbs reach spend # logs and logging callbacks, and these carry either the request payload or router-internal # walk state rather than anything that identifies the failed attempt. @@ -525,14 +537,15 @@ class Router: self.deployment_names: list = [] # names of models under litellm_params. ex. azure/chatgpt-v-2 self.deployment_latency_map = {} ### CACHING ### - cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = "local" # default to an in-memory cache + uses_redis: Final[bool] = redis_url is not None or (redis_host is not None and redis_port is not None) + cache_type: Final[LiteLLMCacheType] = ( + LiteLLMCacheType.REDIS if uses_redis else _resolve_cache_type(cache_kwargs.get("type")) + ) redis_cache = None cache_config: Final[dict[str, Any]] = {} self.client_ttl = client_ttl - if redis_url is not None or (redis_host is not None and redis_port is not None): - cache_type = "redis" - + if uses_redis: if redis_url is not None: cache_config["url"] = redis_url @@ -551,8 +564,9 @@ class Router: ) cache_config["db"] = str(redis_db) - # Add additional key-value pairs from cache_kwargs - cache_config.update(cache_kwargs) + cache_config.update({key: value for key, value in cache_kwargs.items() if key != "type"}) + + if uses_redis: redis_cache = self._create_redis_cache(cache_config) if cache_responses: diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 67fa827a8e4..85eab35344b 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7615,3 +7615,66 @@ def test_ensure_deployment_affinity_callback_is_idempotent(): finally: for cb in router.optional_callbacks or []: litellm.logging_callback_manager.remove_callback_from_all_lists(cb) + +@pytest.fixture +def reset_litellm_cache(): + previous = litellm.cache + litellm.cache = None + try: + yield + finally: + litellm.cache = previous + + +def test_router_cache_kwargs_applied_without_redis(reset_litellm_cache): + litellm.Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}], + cache_responses=True, + cache_kwargs={"ttl": 60, "namespace": "router-ns"}, + ) + + assert litellm.cache is not None + assert litellm.cache.ttl == 60 + assert litellm.cache.namespace == "router-ns" + + +def test_router_cache_kwargs_type_selects_backend_without_redis(tmp_path, reset_litellm_cache): + pytest.importorskip("diskcache") + from litellm.caching.disk_cache import DiskCache + + litellm.Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}], + cache_responses=True, + cache_kwargs={"type": "disk", "disk_cache_dir": str(tmp_path), "ttl": 60}, + ) + + assert litellm.cache is not None + assert isinstance(litellm.cache.cache, DiskCache) + assert litellm.cache.ttl == 60 + + +def test_router_cache_kwargs_type_ignored_when_redis_params_given(reset_litellm_cache): + created_caches = [] + + class _FakeRedisCache: + def __init__(self, **kwargs): + created_caches.append(kwargs) + + with patch("litellm.router.RedisCache", _FakeRedisCache): + litellm.Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}], + redis_host="localhost", + redis_port=6379, + cache_kwargs={"type": "disk", "socket_timeout": 5}, + ) + + assert created_caches == [{"host": "localhost", "port": "6379", "socket_timeout": 5}] + + +def test_router_invalid_cache_kwargs_type_raises(reset_litellm_cache): + with pytest.raises(ValueError): + litellm.Router( + model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}], + cache_responses=True, + cache_kwargs={"type": "not-a-cache"}, + )