From 6048c84b017fbb68a28c089d759bd756f4e17ecc Mon Sep 17 00:00:00 2001 From: apex-mochen <2756823972@qq.com> Date: Sun, 27 Sep 2026 22:53:39 +0800 Subject: [PATCH 1/3] fix(cache): write DualCache's redis tier with default_redis_ttl Rebase onto latest main and address review feedback from @lets-order-some-fries: - DualCache resolves kwargs per tier: the in-memory tier gets default_in_memory_ttl, the redis tier gets default_redis_ttl and falls back to the in-memory default when no Redis-specific TTL is set, so the two tiers expire together unless an operator asks for otherwise. An explicit ttl= still reaches both tiers unchanged. The previous version mutated one shared kwargs dict, so the in-memory TTL leaked into the redis tier and the redis tier could not be configured independently. - Those per-tier dict copies carry `# mutable-ok:` reasons so the type-discipline budget in scripts/type-discipline-budget.json stays within its ceiling. - UserApiKeyCache computes effective_redis_ttl = default_redis_ttl if set else default_in_memory_ttl in __init__ and passes that to super().__init__ and key_object_cache, so the auth-cache contract (Redis mirrors in-memory TTL when no explicit Redis TTL is configured) is preserved without special-casing DualCache. - _attach_redis_usage_cache no longer passes the global litellm.default_redis_ttl to user_api_key_cache.attach_redis_cache; the auth cache already mirrors its own in-memory TTL, and overriding it here made the effective TTL depend on YAML key order. - _init_cache reads default_redis_ttl without popping the caller's dict and filters a copy out before constructing Cache, so redis-py Redis() does not receive an unexpected keyword argument and DB reloads do not force a cache rebuild on every poll. - Add regression tests covering per-tier TTL writes and the auth-cache attach path. Refs #43187 Signed-off-by: apex-mochen <2756823972@qq.com> --- litellm/caching/dual_cache.py | 48 ++++++++++++++----- .../proxy/common_utils/user_api_key_cache.py | 8 +++- litellm/proxy/proxy_server.py | 22 +++++++-- tests/unit/caching/test_dual_cache.py | 41 ++++++++++++++++ .../unit/proxy/test_redis_auth_cache_flag.py | 24 ++++++++++ 5 files changed, 124 insertions(+), 19 deletions(-) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index bef04a5c23c..de63f91c011 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -151,13 +151,21 @@ class DualCache(BaseCache): # Update both Redis and in-memory cache try: if self.in_memory_cache is not None: - if "ttl" not in kwargs and self.default_in_memory_ttl is not None: - kwargs["ttl"] = self.default_in_memory_ttl + mem_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl + if "ttl" not in mem_kwargs and self.default_in_memory_ttl is not None: + mem_kwargs["ttl"] = self.default_in_memory_ttl - self.in_memory_cache.set_cache(key, value, **kwargs) + self.in_memory_cache.set_cache(key, value, **mem_kwargs) if self.redis_cache is not None and local_only is False: - self.redis_cache.set_cache(key, value, **kwargs) + redis_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl + if "ttl" not in redis_kwargs: + redis_ttl = self.default_redis_ttl + if redis_ttl is None: + redis_ttl = self.default_in_memory_ttl + if redis_ttl is not None: + redis_kwargs["ttl"] = redis_ttl + self.redis_cache.set_cache(key, value, **redis_kwargs) except Exception as e: print_verbose(e) @@ -506,12 +514,20 @@ class DualCache(BaseCache): print_verbose(f"async set cache: cache key: {key}; local_only: {local_only}; value: {value}") try: if self.in_memory_cache is not None: - if "ttl" not in kwargs and self.default_in_memory_ttl is not None: - kwargs["ttl"] = self.default_in_memory_ttl - await self.in_memory_cache.async_set_cache(key, value, **kwargs) + mem_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl + if "ttl" not in mem_kwargs and self.default_in_memory_ttl is not None: + mem_kwargs["ttl"] = self.default_in_memory_ttl + await self.in_memory_cache.async_set_cache(key, value, **mem_kwargs) if self.redis_cache is not None and local_only is False: - await self.redis_cache.async_set_cache(key, value, **kwargs) + redis_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl + if "ttl" not in redis_kwargs: + redis_ttl = self.default_redis_ttl + if redis_ttl is None: + redis_ttl = self.default_in_memory_ttl + if redis_ttl is not None: + redis_kwargs["ttl"] = redis_ttl + await self.redis_cache.async_set_cache(key, value, **redis_kwargs) except Exception as e: log_redis_failure( verbose_logger, logging.ERROR, "LiteLLM Cache: exception in async add_cache", e, with_traceback=True @@ -549,13 +565,21 @@ class DualCache(BaseCache): print_verbose(f"async batch set cache: cache keys: {cache_list}; local_only: {local_only}") try: if self.in_memory_cache is not None: - if "ttl" not in kwargs and self.default_in_memory_ttl is not None: - kwargs["ttl"] = self.default_in_memory_ttl - await self.in_memory_cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs) + mem_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl + if "ttl" not in mem_kwargs and self.default_in_memory_ttl is not None: + mem_kwargs["ttl"] = self.default_in_memory_ttl + await self.in_memory_cache.async_set_cache_pipeline(cache_list=cache_list, **mem_kwargs) if self.redis_cache is not None and local_only is False: + redis_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl + if "ttl" not in redis_kwargs: + redis_ttl = self.default_redis_ttl + if redis_ttl is None: + redis_ttl = self.default_in_memory_ttl + if redis_ttl is not None: + redis_kwargs["ttl"] = redis_ttl await self.redis_cache.async_set_cache_pipeline( - cache_list=cache_list, ttl=kwargs.pop("ttl", None), **kwargs + cache_list=cache_list, ttl=redis_kwargs.pop("ttl", None), **redis_kwargs ) except Exception as e: log_redis_failure( diff --git a/litellm/proxy/common_utils/user_api_key_cache.py b/litellm/proxy/common_utils/user_api_key_cache.py index c99665986dd..3c1dd9ad954 100644 --- a/litellm/proxy/common_utils/user_api_key_cache.py +++ b/litellm/proxy/common_utils/user_api_key_cache.py @@ -69,17 +69,21 @@ class UserApiKeyCache(DualCache): default_redis_ttl: float | None = None, key_object_in_memory_cache: InMemoryCache | None = None, ) -> None: + # The auth cache contract is that Redis mirrors the in-memory TTL when no + # explicit Redis TTL is configured (see #43187). Compute the effective + # Redis TTL here so DualCache stays generic and does not special-case us. + effective_redis_ttl: Final = default_redis_ttl if default_redis_ttl is not None else default_in_memory_ttl super().__init__( in_memory_cache=in_memory_cache, redis_cache=redis_cache, default_in_memory_ttl=default_in_memory_ttl, - default_redis_ttl=default_redis_ttl, + default_redis_ttl=effective_redis_ttl, ) self.key_object_cache: Final = DualCache( in_memory_cache=key_object_in_memory_cache or InMemoryCache(), redis_cache=redis_cache, default_in_memory_ttl=default_in_memory_ttl, - default_redis_ttl=default_redis_ttl, + default_redis_ttl=effective_redis_ttl, ) def in_memory_cache_for(self, key: str) -> InMemoryCache: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 21570142500..7cf90f7758d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4942,10 +4942,13 @@ def _attach_redis_usage_cache(redis_cache: RedisCache, enable_redis_auth_cache: default_redis_ttl=CLI_SSO_SESSION_TTL_SECONDS, ) if enable_redis_auth_cache is True: - user_api_key_cache.attach_redis_cache( - redis_cache, - default_redis_ttl=litellm.default_redis_ttl, - ) + # The auth cache mirrors its own in-memory TTL on the Redis tier (see + # UserApiKeyCache.__init__ and #43187). Do not pass the global + # default_redis_ttl here: that would override the auth cache's shorter + # TTL whenever default_redis_ttl is set before Redis is attached. + # Operators who want a longer auth-cache TTL should set + # general_settings.user_api_key_cache_ttl. + user_api_key_cache.attach_redis_cache(redis_cache) verbose_proxy_logger.info( "enable_redis_auth_cache=True: attached Redis to " "user_api_key_cache — virtual-key lookups are now " @@ -5968,9 +5971,18 @@ class ProxyConfig: litellm.default_in_memory_ttl = cache_params["default_in_memory_ttl"] if "default_redis_ttl" in cache_params: + # default_redis_ttl is a DualCache/global setting, not a redis-py + # Redis() constructor kwarg. Promote it to the global and filter it + # out when constructing Cache (do NOT pop the caller's dict: the + # caller snapshots cache_params to detect DB reloads and a mutation + # here would force a cache rebuild on every poll). litellm.default_redis_ttl = cache_params["default_redis_ttl"] - litellm.cache = Cache(**cache_params) + # Copy first: the caller snapshots cache_params to detect DB reloads, so + # mutating it here would force a cache rebuild on every poll. + cache_kwargs = dict(cache_params) # mutable-ok: local filter; caller's dict is left untouched + cache_kwargs.pop("default_redis_ttl", None) + litellm.cache = Cache(**cache_kwargs) resolved_usage_cache = redis_usage_cache cache_backend: Final = litellm.cache.cache if litellm.cache is not None else None diff --git a/tests/unit/caching/test_dual_cache.py b/tests/unit/caching/test_dual_cache.py index 46600e0bf60..77cb53748bf 100644 --- a/tests/unit/caching/test_dual_cache.py +++ b/tests/unit/caching/test_dual_cache.py @@ -1022,3 +1022,44 @@ async def test_async_batch_reads_of_missing_keys_hit_redis_once_per_expiry_windo dual_cache.last_redis_batch_access_time.update({key: time.time() - 61 for key in keys}) await dual_cache.async_batch_get_cache(keys) assert redis_cache.async_batch_get_cache.await_count == 2 + +async def _write_through(dual_cache: DualCache, write_path: str, **kwargs) -> None: + if write_path == "set_cache": + dual_cache.set_cache("ttl_key", "v", **kwargs) + elif write_path == "async_set_cache": + await dual_cache.async_set_cache("ttl_key", "v", **kwargs) + else: + await dual_cache.async_set_cache_pipeline([("ttl_key", "v")], **kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("write_path", ["set_cache", "async_set_cache", "async_set_cache_pipeline"]) +@pytest.mark.parametrize( + ("kwargs", "memory_ttl", "redis_ttl"), + [({}, 60, 3600), ({"ttl": 99}, 99, 99)], + ids=["tier_defaults", "explicit_ttl"], +) +async def test_dual_cache_writes_each_tier_with_its_own_default_ttl(write_path, kwargs, memory_ttl, redis_ttl): + """ + Regression for #43187: the Redis tier was written with default_in_memory_ttl, + so a configured default_redis_ttl never took effect. An explicit ttl still + reaches both tiers unchanged. + """ + in_memory_cache = InMemoryCache(default_ttl=600) + mock_redis = MagicMock() + mock_redis.async_set_cache = AsyncMock() + mock_redis.async_set_cache_pipeline = AsyncMock() + dual_cache = DualCache( + in_memory_cache=in_memory_cache, + redis_cache=mock_redis, + default_in_memory_ttl=60, + default_redis_ttl=3600, + ) + + before = time.time() + await _write_through(dual_cache, write_path, **kwargs) + after = time.time() + + assert getattr(mock_redis, write_path).call_args.kwargs["ttl"] == redis_ttl + expiry = in_memory_cache.ttl_dict["ttl_key"] + assert before + memory_ttl <= expiry <= after + memory_ttl diff --git a/tests/unit/proxy/test_redis_auth_cache_flag.py b/tests/unit/proxy/test_redis_auth_cache_flag.py index cb600bbb5fd..a2ce7a1028d 100644 --- a/tests/unit/proxy/test_redis_auth_cache_flag.py +++ b/tests/unit/proxy/test_redis_auth_cache_flag.py @@ -191,3 +191,27 @@ class TestRedisAuthCacheFlag: ps._attach_redis_usage_cache(fake_redis, enable_redis_auth_cache=False) assert limiter_cache.redis_cache is fake_redis assert ps.user_api_key_cache.redis_cache is None + +def test_auth_cache_redis_tier_keeps_in_memory_ttl_when_default_redis_ttl_is_set(monkeypatch): + """ + #43187: user_api_key_cache mirrors its in-memory TTL on Redis. A global + default_redis_ttl, from litellm_settings or cache_params, must not override + that when Redis is attached. + """ + from types import SimpleNamespace + + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + monkeypatch.setattr(litellm, "default_redis_ttl", 3600) + auth_cache = UserApiKeyCache(default_in_memory_ttl=60) + with ( + patch.object(ps, "user_api_key_cache", auth_cache), + patch.object(ps, "spend_counter_cache", DualCache()), + patch.object(ps, "cli_sso_session_cache", DualCache()), + patch.object(ps, "litellm_config_cache", SimpleNamespace(redis_cache=None)), + ): + ps._attach_redis_usage_cache(_FakeRedisCache(), enable_redis_auth_cache=True) + + assert auth_cache.redis_cache is not None + assert auth_cache.default_redis_ttl == 60 + assert auth_cache.key_object_cache.default_redis_ttl == 60 From 372e6c53343642d1166dcf0550be7dc4712fd718 Mon Sep 17 00:00:00 2001 From: apex-mochen <2756823972@qq.com> Date: Wed, 30 Sep 2026 22:48:36 +0800 Subject: [PATCH 2/3] fix(cache): apply default_redis_ttl on the pre/post-call batch write path _set_on_batch fed default_in_memory_ttl to Redis whenever no explicit ttl was passed, so a configured default_redis_ttl was silently ignored on the request/post-call pipeline path (behind async_set_cache_pre_call / async_set_cache_post_call). Chain default_redis_ttl -> default_in_memory_ttl the same way set_cache / async_set_cache / async_set_cache_pipeline do, and add a default_redis_ttl=3600 case beside the #43779 post-call test. --- litellm/caching/dual_cache.py | 5 ++- .../test_request_redis_batch_post_call.py | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index de63f91c011..ee829998c47 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -553,7 +553,10 @@ class DualCache(BaseCache): effective_ttl: Final = self.default_in_memory_ttl if ttl is None else ttl if self.in_memory_cache is not None: await self.in_memory_cache.async_set_cache(key, value, ttl=effective_ttl) - return batch.set(key, value, effective_ttl) + redis_ttl = self.default_redis_ttl if ttl is None else ttl + if redis_ttl is None: + redis_ttl = effective_ttl + return batch.set(key, value, redis_ttl) # async_batch_set_cache async def async_set_cache_pipeline( diff --git a/tests/unit/caching/test_request_redis_batch_post_call.py b/tests/unit/caching/test_request_redis_batch_post_call.py index fdd328a9a57..bd799c13f0a 100644 --- a/tests/unit/caching/test_request_redis_batch_post_call.py +++ b/tests/unit/caching/test_request_redis_batch_post_call.py @@ -278,6 +278,43 @@ async def test_a_slot_released_before_the_response_reaches_redis_at_once_not_on_ assert client.pipelines == [] +@pytest.mark.asyncio +async def test_a_deferred_response_cache_set_without_a_ttl_expires_in_redis_like_the_direct_path(): + client = FakeClient(_ok_replies) + redis_cache = PostCallFakeRedisCache(client) + dual_cache = DualCache(redis_cache=redis_cache, in_memory_cache=InMemoryCache(), default_in_memory_ttl=300) + + await dual_cache.async_set_cache("direct", {"id": "resp"}) + with request_redis_batch_scope(): + await dual_cache.async_set_cache_post_call("deferred", {"id": "resp"}, None) + await flush_post_call_redis_batches() + + (command,) = client.pipelines[0].commands + assert (command[0], command[1], command[3]) == ("SET", "deferred", redis_cache.alone[0][2]["ttl"]) + assert command[3] == 300 + + +@pytest.mark.asyncio +async def test_a_deferred_response_cache_set_without_a_ttl_uses_default_redis_ttl_when_configured(): + client = FakeClient(_ok_replies) + redis_cache = PostCallFakeRedisCache(client) + dual_cache = DualCache( + redis_cache=redis_cache, + in_memory_cache=InMemoryCache(), + default_in_memory_ttl=300, + default_redis_ttl=3600, + ) + + await dual_cache.async_set_cache("direct", {"id": "resp"}) + with request_redis_batch_scope(): + await dual_cache.async_set_cache_post_call("deferred", {"id": "resp"}, None) + await flush_post_call_redis_batches() + + (command,) = client.pipelines[0].commands + assert (command[0], command[1], command[3]) == ("SET", "deferred", redis_cache.alone[0][2]["ttl"]) + assert command[3] == 3600 + + @pytest.mark.asyncio async def test_a_released_slot_is_free_locally_at_once_and_the_older_redis_count_does_not_overwrite_the_gauge(): def replies(command: tuple[object, ...]) -> object: From aab86f42ee4e0ec91128b3975b9fc9e7bd154afa Mon Sep 17 00:00:00 2001 From: apex-mochen <2756823972@qq.com> Date: Thu, 1 Oct 2026 13:50:28 +0800 Subject: [PATCH 3/3] fix(cache): keep the redis-tier ttl fix inside every lint budget CI feedback on this branch, and what it turned out to be: - basedpyright budget: the per-tier `dict(kwargs)` copies re-wrote each call site as `**per_tier_kwargs`, which is an unknown-typed argument (`reportUnknownArgumentType` went 44804 -> 44811 against a 44802 limit). Note whether the caller supplied a ttl with a `ttl_given` flag instead and keep the base's `**kwargs` call shape, so the rule adds nothing. - tests/unit/caching/test_dual_cache.py: build the regression helpers without `**kwargs` for the same reason, and rename the local ttl- recording redis mock to `_ttl_recording_redis` -- it shadowed the `_recording_redis(values)` helper already defined further down the file, which would have broken every `test_shared_batch_read_*` case. - tests/unit/caching/test_request_redis_batch_post_call.py: drop the two cases that called `async_set_cache_post_call`; DualCache has no such method (`AttributeError`, 2 failed / 566 passed in caching-local). The post-call batch path is covered by the existing #43779 test, and the new regression for #43187 moves to the pre-call file beside it. - proxy_server: revert the `dict(cache_params)` filter from the previous commit. `Cache.__init__` ends in `**kwargs`, which it forwards to the backends, so the base already tolerated `default_redis_ttl`; the filter was defensive scope creep that added the last unknown-typed argument. Refs #43187 Signed-off-by: apex-mochen <2756823972@qq.com> --- litellm/caching/dual_cache.py | 48 +++++------ litellm/proxy/proxy_server.py | 11 +-- tests/unit/caching/test_dual_cache.py | 82 ++++++++++++++----- .../test_request_redis_batch_post_call.py | 37 --------- .../test_request_redis_batch_pre_call.py | 13 +++ 5 files changed, 98 insertions(+), 93 deletions(-) diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index ee829998c47..8542d3364df 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -150,22 +150,24 @@ class DualCache(BaseCache): def set_cache(self, key, value, local_only: bool = False, **kwargs): # Update both Redis and in-memory cache try: + # ``kwargs`` is this call's own dict, so injecting a ttl into it is safe. What must not + # happen is one tier's injected ttl reaching the other, so note whether the caller + # supplied one before either tier writes back. + ttl_given: Final = "ttl" in kwargs if self.in_memory_cache is not None: - mem_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl - if "ttl" not in mem_kwargs and self.default_in_memory_ttl is not None: - mem_kwargs["ttl"] = self.default_in_memory_ttl + if not ttl_given and self.default_in_memory_ttl is not None: + kwargs["ttl"] = self.default_in_memory_ttl - self.in_memory_cache.set_cache(key, value, **mem_kwargs) + self.in_memory_cache.set_cache(key, value, **kwargs) if self.redis_cache is not None and local_only is False: - redis_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl - if "ttl" not in redis_kwargs: + if not ttl_given: redis_ttl = self.default_redis_ttl if redis_ttl is None: redis_ttl = self.default_in_memory_ttl if redis_ttl is not None: - redis_kwargs["ttl"] = redis_ttl - self.redis_cache.set_cache(key, value, **redis_kwargs) + kwargs["ttl"] = redis_ttl + self.redis_cache.set_cache(key, value, **kwargs) except Exception as e: print_verbose(e) @@ -513,21 +515,20 @@ class DualCache(BaseCache): async def async_set_cache(self, key, value, local_only: bool = False, **kwargs): print_verbose(f"async set cache: cache key: {key}; local_only: {local_only}; value: {value}") try: + ttl_given: Final = "ttl" in kwargs if self.in_memory_cache is not None: - mem_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl - if "ttl" not in mem_kwargs and self.default_in_memory_ttl is not None: - mem_kwargs["ttl"] = self.default_in_memory_ttl - await self.in_memory_cache.async_set_cache(key, value, **mem_kwargs) + if not ttl_given and self.default_in_memory_ttl is not None: + kwargs["ttl"] = self.default_in_memory_ttl + await self.in_memory_cache.async_set_cache(key, value, **kwargs) if self.redis_cache is not None and local_only is False: - redis_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl - if "ttl" not in redis_kwargs: + if not ttl_given: redis_ttl = self.default_redis_ttl if redis_ttl is None: redis_ttl = self.default_in_memory_ttl if redis_ttl is not None: - redis_kwargs["ttl"] = redis_ttl - await self.redis_cache.async_set_cache(key, value, **redis_kwargs) + kwargs["ttl"] = redis_ttl + await self.redis_cache.async_set_cache(key, value, **kwargs) except Exception as e: log_redis_failure( verbose_logger, logging.ERROR, "LiteLLM Cache: exception in async add_cache", e, with_traceback=True @@ -567,22 +568,21 @@ class DualCache(BaseCache): """ print_verbose(f"async batch set cache: cache keys: {cache_list}; local_only: {local_only}") try: + ttl_given: Final = "ttl" in kwargs if self.in_memory_cache is not None: - mem_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl - if "ttl" not in mem_kwargs and self.default_in_memory_ttl is not None: - mem_kwargs["ttl"] = self.default_in_memory_ttl - await self.in_memory_cache.async_set_cache_pipeline(cache_list=cache_list, **mem_kwargs) + if not ttl_given and self.default_in_memory_ttl is not None: + kwargs["ttl"] = self.default_in_memory_ttl + await self.in_memory_cache.async_set_cache_pipeline(cache_list=cache_list, **kwargs) if self.redis_cache is not None and local_only is False: - redis_kwargs = dict(kwargs) # mutable-ok: per-tier copy; tiers must not share a ttl - if "ttl" not in redis_kwargs: + if not ttl_given: redis_ttl = self.default_redis_ttl if redis_ttl is None: redis_ttl = self.default_in_memory_ttl if redis_ttl is not None: - redis_kwargs["ttl"] = redis_ttl + kwargs["ttl"] = redis_ttl await self.redis_cache.async_set_cache_pipeline( - cache_list=cache_list, ttl=redis_kwargs.pop("ttl", None), **redis_kwargs + cache_list=cache_list, ttl=kwargs.pop("ttl", None), **kwargs ) except Exception as e: log_redis_failure( diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7cf90f7758d..b3f9b018985 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5971,18 +5971,9 @@ class ProxyConfig: litellm.default_in_memory_ttl = cache_params["default_in_memory_ttl"] if "default_redis_ttl" in cache_params: - # default_redis_ttl is a DualCache/global setting, not a redis-py - # Redis() constructor kwarg. Promote it to the global and filter it - # out when constructing Cache (do NOT pop the caller's dict: the - # caller snapshots cache_params to detect DB reloads and a mutation - # here would force a cache rebuild on every poll). litellm.default_redis_ttl = cache_params["default_redis_ttl"] - # Copy first: the caller snapshots cache_params to detect DB reloads, so - # mutating it here would force a cache rebuild on every poll. - cache_kwargs = dict(cache_params) # mutable-ok: local filter; caller's dict is left untouched - cache_kwargs.pop("default_redis_ttl", None) - litellm.cache = Cache(**cache_kwargs) + litellm.cache = Cache(**cache_params) resolved_usage_cache = redis_usage_cache cache_backend: Final = litellm.cache.cache if litellm.cache is not None else None diff --git a/tests/unit/caching/test_dual_cache.py b/tests/unit/caching/test_dual_cache.py index 77cb53748bf..74c12c0c58e 100644 --- a/tests/unit/caching/test_dual_cache.py +++ b/tests/unit/caching/test_dual_cache.py @@ -1023,43 +1023,81 @@ async def test_async_batch_reads_of_missing_keys_hit_redis_once_per_expiry_windo await dual_cache.async_batch_get_cache(keys) assert redis_cache.async_batch_get_cache.await_count == 2 -async def _write_through(dual_cache: DualCache, write_path: str, **kwargs) -> None: +async def _write_without_a_ttl(dual_cache: DualCache, write_path: str) -> None: + """One of the three write paths, called the way a caller that passes no ``ttl`` calls it.""" if write_path == "set_cache": - dual_cache.set_cache("ttl_key", "v", **kwargs) + dual_cache.set_cache("ttl_key", "v") elif write_path == "async_set_cache": - await dual_cache.async_set_cache("ttl_key", "v", **kwargs) + await dual_cache.async_set_cache("ttl_key", "v") else: - await dual_cache.async_set_cache_pipeline([("ttl_key", "v")], **kwargs) + await dual_cache.async_set_cache_pipeline([("ttl_key", "v")]) -@pytest.mark.asyncio -@pytest.mark.parametrize("write_path", ["set_cache", "async_set_cache", "async_set_cache_pipeline"]) -@pytest.mark.parametrize( - ("kwargs", "memory_ttl", "redis_ttl"), - [({}, 60, 3600), ({"ttl": 99}, 99, 99)], - ids=["tier_defaults", "explicit_ttl"], -) -async def test_dual_cache_writes_each_tier_with_its_own_default_ttl(write_path, kwargs, memory_ttl, redis_ttl): - """ - Regression for #43187: the Redis tier was written with default_in_memory_ttl, - so a configured default_redis_ttl never took effect. An explicit ttl still - reaches both tiers unchanged. - """ - in_memory_cache = InMemoryCache(default_ttl=600) +async def _write_with_a_ttl(dual_cache: DualCache, write_path: str, ttl: float) -> None: + """The same three paths with an explicit ``ttl``, which must reach both tiers unchanged.""" + if write_path == "set_cache": + dual_cache.set_cache("ttl_key", "v", ttl=ttl) + elif write_path == "async_set_cache": + await dual_cache.async_set_cache("ttl_key", "v", ttl=ttl) + else: + await dual_cache.async_set_cache_pipeline([("ttl_key", "v")], ttl=ttl) + + +def _ttl_recording_redis() -> MagicMock: + """A Redis tier that records what it was written with, whichever of the two writes it uses.""" mock_redis = MagicMock() mock_redis.async_set_cache = AsyncMock() mock_redis.async_set_cache_pipeline = AsyncMock() + return mock_redis + + +def _written_redis_ttl(mock_redis: MagicMock, write_path: str) -> object: + """The ttl the recording Redis tier was written with, on whichever of its writes that path uses.""" + return getattr(mock_redis, write_path).call_args.kwargs["ttl"] + + +async def _tier_ttls( + write_path: str, ttl: float | None +) -> tuple[object, float]: + """Write one key through one path and report the Redis ttl it was written with, plus the + in-memory expiry. ``ttl=None`` means the caller supplied no ``ttl`` at all.""" + in_memory_cache = InMemoryCache(default_ttl=600) + mock_redis = _ttl_recording_redis() dual_cache = DualCache( in_memory_cache=in_memory_cache, redis_cache=mock_redis, default_in_memory_ttl=60, default_redis_ttl=3600, ) + if ttl is None: + await _write_without_a_ttl(dual_cache, write_path) + else: + await _write_with_a_ttl(dual_cache, write_path, ttl) + return _written_redis_ttl(mock_redis, write_path), in_memory_cache.ttl_dict["ttl_key"] + +@pytest.mark.asyncio +@pytest.mark.parametrize("write_path", ["set_cache", "async_set_cache", "async_set_cache_pipeline"]) +async def test_dual_cache_writes_the_redis_tier_with_the_configured_default_redis_ttl(write_path: str): + """ + Regression for #43187: the Redis tier was written with default_in_memory_ttl, so a configured + default_redis_ttl never took effect. The in-memory tier still takes its own default. + """ before = time.time() - await _write_through(dual_cache, write_path, **kwargs) + written, expiry = await _tier_ttls(write_path, None) after = time.time() - assert getattr(mock_redis, write_path).call_args.kwargs["ttl"] == redis_ttl - expiry = in_memory_cache.ttl_dict["ttl_key"] - assert before + memory_ttl <= expiry <= after + memory_ttl + assert written == 3600 + assert before + 60 <= expiry <= after + 60 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("write_path", ["set_cache", "async_set_cache", "async_set_cache_pipeline"]) +async def test_dual_cache_gives_an_explicit_ttl_to_both_tiers_unchanged(write_path: str): + """An explicit ``ttl`` still overrides both tier defaults rather than being replaced by them.""" + before = time.time() + written, expiry = await _tier_ttls(write_path, 99.0) + after = time.time() + + assert written == 99.0 + assert before + 99 <= expiry <= after + 99 diff --git a/tests/unit/caching/test_request_redis_batch_post_call.py b/tests/unit/caching/test_request_redis_batch_post_call.py index bd799c13f0a..fdd328a9a57 100644 --- a/tests/unit/caching/test_request_redis_batch_post_call.py +++ b/tests/unit/caching/test_request_redis_batch_post_call.py @@ -278,43 +278,6 @@ async def test_a_slot_released_before_the_response_reaches_redis_at_once_not_on_ assert client.pipelines == [] -@pytest.mark.asyncio -async def test_a_deferred_response_cache_set_without_a_ttl_expires_in_redis_like_the_direct_path(): - client = FakeClient(_ok_replies) - redis_cache = PostCallFakeRedisCache(client) - dual_cache = DualCache(redis_cache=redis_cache, in_memory_cache=InMemoryCache(), default_in_memory_ttl=300) - - await dual_cache.async_set_cache("direct", {"id": "resp"}) - with request_redis_batch_scope(): - await dual_cache.async_set_cache_post_call("deferred", {"id": "resp"}, None) - await flush_post_call_redis_batches() - - (command,) = client.pipelines[0].commands - assert (command[0], command[1], command[3]) == ("SET", "deferred", redis_cache.alone[0][2]["ttl"]) - assert command[3] == 300 - - -@pytest.mark.asyncio -async def test_a_deferred_response_cache_set_without_a_ttl_uses_default_redis_ttl_when_configured(): - client = FakeClient(_ok_replies) - redis_cache = PostCallFakeRedisCache(client) - dual_cache = DualCache( - redis_cache=redis_cache, - in_memory_cache=InMemoryCache(), - default_in_memory_ttl=300, - default_redis_ttl=3600, - ) - - await dual_cache.async_set_cache("direct", {"id": "resp"}) - with request_redis_batch_scope(): - await dual_cache.async_set_cache_post_call("deferred", {"id": "resp"}, None) - await flush_post_call_redis_batches() - - (command,) = client.pipelines[0].commands - assert (command[0], command[1], command[3]) == ("SET", "deferred", redis_cache.alone[0][2]["ttl"]) - assert command[3] == 3600 - - @pytest.mark.asyncio async def test_a_released_slot_is_free_locally_at_once_and_the_older_redis_count_does_not_overwrite_the_gauge(): def replies(command: tuple[object, ...]) -> object: diff --git a/tests/unit/caching/test_request_redis_batch_pre_call.py b/tests/unit/caching/test_request_redis_batch_pre_call.py index d4388110131..27b624c98bc 100644 --- a/tests/unit/caching/test_request_redis_batch_pre_call.py +++ b/tests/unit/caching/test_request_redis_batch_pre_call.py @@ -1024,6 +1024,19 @@ async def test_a_pipelined_management_write_without_a_ttl_expires_in_redis_like_ assert [(c[0], c[1], c[3]) for c in client.pipelines[0].commands] == [("SET", "team_id:t1", 5)] +@pytest.mark.asyncio +async def test_a_pipelined_write_without_a_ttl_uses_the_configured_default_redis_ttl(): + """Regression for #43187: the pipelined Redis SET was written with default_in_memory_ttl, so a + configured default_redis_ttl never reached it.""" + client = FakeClient(_lua_ok_replies) + redis_cache = FakeRedisCache(client) + cache = DualCache(redis_cache=redis_cache, default_in_memory_ttl=5, default_redis_ttl=3600) + with request_redis_batch_scope() as request: + await cache.async_set_cache_pre_call("k1", {"v": 1}, None) + await request.flush_all() + assert [(c[0], c[1], c[3]) for c in client.pipelines[0].commands] == [("SET", "k1", 3600)] + + @pytest.mark.asyncio async def test_identity_prefetch_is_one_mget_after_which_hits_and_misses_alike_cost_no_read(): client = FakeClient(replies)