From 438ad98928dbd85db105aef66e5f567c51a34035 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:47:38 -0700 Subject: [PATCH] test(caching): cover the cache key fallback for loops without weak references Also corrects the evicted client closer docstring: a dropped bucket's clients are left to the garbage collector. --- litellm/caching/evicted_client_closer.py | 3 ++- .../unit/caching/test_llm_caching_handler.py | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/litellm/caching/evicted_client_closer.py b/litellm/caching/evicted_client_closer.py index 523fbe106f7..6f29e224249 100644 --- a/litellm/caching/evicted_client_closer.py +++ b/litellm/caching/evicted_client_closer.py @@ -29,7 +29,8 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to close them, and each bucket is ordered by deadline, so a reap walks the entries that are due rather than the whole queue. -A bucket whose loop has been collected is dropped, since nothing can await its closes any more. +A bucket whose loop has been collected is dropped, and its clients are left to the +collector, as they are when the queue is full. The queue holds its clients weakly, so waiting out a grace window never keeps alive anything the collector would have reclaimed first. diff --git a/tests/unit/caching/test_llm_caching_handler.py b/tests/unit/caching/test_llm_caching_handler.py index c81b4ebd9f4..465ba2f9150 100644 --- a/tests/unit/caching/test_llm_caching_handler.py +++ b/tests/unit/caching/test_llm_caching_handler.py @@ -102,6 +102,29 @@ def test_async_client_cache_keys_differ_for_every_collected_loop() -> None: assert len(set(keys)) == len(keys) +class _LoopWithoutWeakReferences: + __slots__ = () + + +def _cache_key_while_running(cache: LLMClientCache, loop: _LoopWithoutWeakReferences) -> str: + asyncio._set_running_loop(loop) + try: + return cache.update_cache_key_with_event_loop("client") + finally: + asyncio._set_running_loop(None) + + +def test_cache_keys_stay_per_loop_on_loops_without_weak_references() -> None: + cache: Final = LLMClientCache() + loop_a: Final = _LoopWithoutWeakReferences() + loop_b: Final = _LoopWithoutWeakReferences() + + first_key_on_a: Final = _cache_key_while_running(cache, loop_a) + + assert _cache_key_while_running(cache, loop_a) == first_key_on_a + assert _cache_key_while_running(cache, loop_b) != first_key_on_a + + @pytest.mark.asyncio async def test_remove_key_does_not_close_async_client(): """