From 54b7f17ca6757bfa3cea81ef4ab228875438accf Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Sun, 2 Mar 2025 08:39:06 -0800 Subject: [PATCH] =?UTF-8?q?fix(proxy=5Fserver.py):=20fix=20setting=20route?= =?UTF-8?q?r=20redis=20cache,=20if=20cache=20enable=E2=80=A6=20(#8859)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(proxy_server.py): fix setting router redis cache, if cache enabled on litellm_settings enables configurations like namespace to just work * fix(redis_cache.py): fix key for async increment, to use the set namespace prevents collisions if redis instance shared across environments * fix load tests on litellm release notes * fix caching on main branch (#8858) * fix(streaming_handler.py): fix is delta empty check to handle empty str * fix(streaming_handler.py): fix delta chunk on final response * [Bug]: Deepseek error on proxy after upgrading to 1.61.13-stable (#8860) * fix deepseek error * test_deepseek_provider_async_completion * fix get_complete_url * bump: version 1.61.17 → 1.61.18 * bump: version 1.61.18 → 1.61.19 * vertex ai anthropic thinking param support (#8853) * fix(vertex_llm_base.py): handle credentials passed in as dictionary * fix(router.py): support vertex credentials as json dict * test(test_vertex.py): allows easier testing mock anthropic thinking response for vertex ai * test(vertex_ai_partner_models/): don't remove "@" from model breaks anthropic cost calculation * test: move testing * fix: fix linting error * fix: fix linting error * fix(vertex_ai_partner_models/main.py): split @ for codestral model * test: fix test * fix: fix stripping "@" on mistral models * fix: fix test * test: fix test --------- Co-authored-by: Ishaan Jaff --- litellm/caching/redis_cache.py | 1 + litellm/proxy/_new_secret_config.yaml | 5 +++ litellm/proxy/proxy_server.py | 7 ++-- tests/litellm/caching/test_redis_cache.py | 41 +++++++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 tests/litellm/caching/test_redis_cache.py diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index 960d19c3f81..66245e7476d 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -543,6 +543,7 @@ class RedisCache(BaseCache): _redis_client: Redis = self.init_async_client() # type: ignore start_time = time.time() _used_ttl = self.get_ttl(ttl=ttl) + key = self.check_and_fix_namespace(key=key) try: result = await _redis_client.incrbyfloat(name=key, amount=value) if _used_ttl is not None: diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index c3be5aa1d99..649d5a25d0a 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -27,3 +27,8 @@ model_list: litellm_params: model: openai/gpt-4o +litellm_settings: + cache: true + cache_params: # set cache params for redis + type: redis + namespace: "litellm.caching" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 68fee62c760..19d0ebe9ea4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1653,10 +1653,6 @@ class ProxyConfig: ## INIT PROXY REDIS USAGE CLIENT ## redis_usage_cache = litellm.cache.cache - ## INIT ROUTER REDIS CACHE ## - if llm_router is not None: - llm_router._update_redis_cache(cache=redis_usage_cache) - async def get_config(self, config_file_path: Optional[str] = None) -> dict: """ Load config file @@ -2183,6 +2179,9 @@ class ProxyConfig: ), ) # type:ignore + if redis_usage_cache is not None and router.cache.redis_cache is None: + router._update_redis_cache(cache=redis_usage_cache) + # Guardrail settings guardrails_v2: Optional[List[Dict]] = None diff --git a/tests/litellm/caching/test_redis_cache.py b/tests/litellm/caching/test_redis_cache.py new file mode 100644 index 00000000000..c2549d7fce7 --- /dev/null +++ b/tests/litellm/caching/test_redis_cache.py @@ -0,0 +1,41 @@ +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path +from unittest.mock import AsyncMock + +from litellm.caching.redis_cache import RedisCache + + +@pytest.mark.parametrize("namespace", [None, "test"]) +@pytest.mark.asyncio +async def test_redis_cache_async_increment(namespace): + redis_cache = RedisCache(namespace=namespace) + # Create an AsyncMock for the Redis client + mock_redis_instance = AsyncMock() + + # Make sure the mock can be used as an async context manager + mock_redis_instance.__aenter__.return_value = mock_redis_instance + mock_redis_instance.__aexit__.return_value = None + + assert redis_cache is not None + + expected_key = "test:test" if namespace else "test" + + with patch.object( + redis_cache, "init_async_client", return_value=mock_redis_instance + ): + # Call async_set_cache + await redis_cache.async_increment(key=expected_key, value=1) + + # Verify that the set method was called on the mock Redis instance + mock_redis_instance.incrbyfloat.assert_called_once_with( + name=expected_key, amount=1 + )