From 2735bd67fcd0884e235e923733cb70a6e946840f Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 16 Oct 2026 21:36:37 +0000 Subject: [PATCH 1/2] fix(caching): key cached async clients by a per-loop token instead of id(loop) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/evicted_client_closer.py | 34 +++++++--- litellm/caching/llm_caching_handler.py | 11 +-- .../caching/test_evicted_client_closer.py | 68 +++++++++++++++++++ .../unit/caching/test_llm_caching_handler.py | 28 ++++++++ 4 files changed, 123 insertions(+), 18 deletions(-) diff --git a/litellm/caching/evicted_client_closer.py b/litellm/caching/evicted_client_closer.py index b22cd3aecd1..bfd1edcccf9 100644 --- a/litellm/caching/evicted_client_closer.py +++ b/litellm/caching/evicted_client_closer.py @@ -36,6 +36,7 @@ alive anything the collector would have reclaimed first. import asyncio import contextlib import inspect +import itertools import threading import time import weakref @@ -53,6 +54,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere" _CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop" _BucketKey = str | int +_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary() +_LOOP_TOKEN_COUNTER: Final = itertools.count() +_LOOP_TOKEN_LOCK: Final = threading.Lock() @dataclass(frozen=True, slots=True) @@ -63,12 +67,12 @@ class _PendingClose: collector would otherwise have reclaimed first. ``needs_loop`` is set for a client whose close is a coroutine; those can only - be closed from the event loop they were evicted on, recorded in ``loop_id``. + be closed from the event loop they were evicted on, recorded in ``loop_token``. A client that closes synchronously carries neither constraint. """ client_ref: "weakref.ref[object]" - loop_id: int | None + loop_token: int | None needs_loop: bool close_after: float @@ -77,16 +81,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey: """Which reaps can close this entry: any at all, any running a loop, or one loop's.""" if not pending.needs_loop: return _CLOSABLE_ANYWHERE - if pending.loop_id is None: + if pending.loop_token is None: return _CLOSABLE_ON_ANY_LOOP - return pending.loop_id + return pending.loop_token -def _running_loop_id() -> int | None: +def running_loop_token() -> int | None: try: - return id(asyncio.get_running_loop()) + loop: Final = asyncio.get_running_loop() except RuntimeError: return None + try: + token: Final[int | None] = _LOOP_TOKENS.get(loop) + if token is not None: + return token + with _LOOP_TOKEN_LOCK: + return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER)) + except TypeError: + return id(loop) def _close_function(client: object) -> Callable[[], object] | None: @@ -202,7 +214,7 @@ class EvictedClientCloser: self._enqueue( _PendingClose( client_ref=weakref.ref(client), - loop_id=_running_loop_id(), + loop_token=running_loop_token(), needs_loop=inspect.iscoroutinefunction(close_fn), close_after=self._clock() + self._grace_seconds, ) @@ -217,7 +229,7 @@ class EvictedClientCloser: if not self._pending_count: return now: Final = self._clock() - for pending in self._take_due(_running_loop_id(), now): + for pending in self._take_due(running_loop_token(), now): client = pending.client_ref() if client is None: continue @@ -245,8 +257,10 @@ class EvictedClientCloser: bucket.append(pending) self._pending_count += 1 - def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]: - buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id) + def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]: + buckets = ( + (_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token) + ) with self._queue_lock: return tuple(pending for key in buckets for pending in self._drain_locked(key, now)) diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index a89e43b78b4..beb9301e570 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -2,10 +2,9 @@ Add the event loop to the cache key, to prevent event loop closed errors. """ -import asyncio from typing import Final -from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer +from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token from .in_memory_cache import InMemoryCache @@ -48,12 +47,8 @@ class LLMClientCache(InMemoryCache): Add the event loop to the cache key, to prevent event loop closed errors. If none, use the key as is. """ - try: - event_loop: Final = asyncio.get_running_loop() - stringified_event_loop: Final = str(id(event_loop)) - return f"{key}-{stringified_event_loop}" - except RuntimeError: # handle no current running event loop - return key + token: Final = running_loop_token() + return f"{key}-{token}" if token is not None else key def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs): """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted.""" diff --git a/tests/unit/caching/test_evicted_client_closer.py b/tests/unit/caching/test_evicted_client_closer.py index 7679e276621..cfb6ded02fd 100644 --- a/tests/unit/caching/test_evicted_client_closer.py +++ b/tests/unit/caching/test_evicted_client_closer.py @@ -10,6 +10,7 @@ never closed, because litellm does not own its lifecycle. import asyncio import gc import weakref +from typing import Final from unittest.mock import AsyncMock import httpx @@ -41,6 +42,14 @@ class AsyncClient: self.closed = True +class RecordingAsyncClient: + def __init__(self) -> None: + self.close_calls = 0 + + async def aclose(self) -> None: + self.close_calls += 1 + + class SyncClient: def __init__(self) -> None: self.closed = False @@ -74,6 +83,40 @@ def make_closer(clock: FakeClock, grace_seconds: float = 60.0) -> EvictedClientC return EvictedClientCloser(grace_seconds=grace_seconds, clock=clock) +async def _schedule_client(closer: EvictedClientCloser, client: RecordingAsyncClient) -> None: + closer.schedule(client) + + +async def _reap_and_yield(closer: EvictedClientCloser) -> None: + closer.reap() + await asyncio.sleep(0) + + +def _reap_after_loop_id_reuse( + closer: EvictedClientCloser, + client: RecordingAsyncClient, + loop_ids: frozenset[int], + attempts: int, +) -> bool: + if attempts == 0: + return False + loop: Final = asyncio.new_event_loop() + loop_id: Final = id(loop) + if loop_id in loop_ids: + try: + loop.run_until_complete(_reap_and_yield(closer)) + finally: + loop.close() + return True + try: + loop.run_until_complete(_schedule_client(closer, client)) + finally: + loop.close() + del loop + gc.collect() + return _reap_after_loop_id_reuse(closer, client, loop_ids | {loop_id}, attempts - 1) + + @pytest.mark.asyncio async def test_redis_client_is_closed_only_after_its_subscription_releases_the_connection(): closer = EvictedClientCloser(grace_seconds=0) @@ -266,6 +309,31 @@ def test_sync_client_evicted_outside_an_event_loop_is_still_closed(): assert closer.pending_count == 0 +def test_async_client_from_collected_loop_is_not_closed_by_a_loop_reusing_its_id() -> None: + clock: Final = FakeClock() + closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock) + client: Final = RecordingAsyncClient() + closer.mark_owned(client) + loop_a: Final = asyncio.new_event_loop() + loop_a_id: Final = id(loop_a) + + async def schedule() -> None: + closer.schedule(client) + + try: + loop_a.run_until_complete(schedule()) + finally: + loop_a.close() + del loop_a + gc.collect() + assert closer.pending_count == 1 + + reused_loop_id: Final = _reap_after_loop_id_reuse(closer, client, frozenset({loop_a_id}), attempts=50) + assert reused_loop_id, "new event loops did not reuse a collected loop's id" + + assert client.close_calls == 0 + + @pytest.mark.asyncio async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): clock = FakeClock() diff --git a/tests/unit/caching/test_llm_caching_handler.py b/tests/unit/caching/test_llm_caching_handler.py index 0c0fe8ca3c5..b0dc53baf11 100644 --- a/tests/unit/caching/test_llm_caching_handler.py +++ b/tests/unit/caching/test_llm_caching_handler.py @@ -10,6 +10,7 @@ See: https://github.com/BerriAI/litellm/pull/22247 import asyncio, gc, httpx, importlib, litellm, os, threading, time, weakref import warnings +from typing import Final import pytest @@ -28,6 +29,24 @@ from litellm.utils import _invalidate_model_cost_lowercase_map from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome +async def _get_and_set_cache(cache: LLMClientCache) -> object | None: + cached: Final = await cache.async_get_cache("client") + await cache.async_set_cache("client", object()) + return cached + + +def _cache_result_on_new_loop(cache: LLMClientCache) -> tuple[int, object | None]: + loop: Final = asyncio.new_event_loop() + loop_id: Final = id(loop) + try: + result: Final = loop.run_until_complete(_get_and_set_cache(cache)) + finally: + loop.close() + del loop + gc.collect() + return loop_id, result + + class MockAsyncClient: """Mock async HTTP client with an async close method.""" @@ -48,6 +67,15 @@ class MockSyncClient: self.closed = True +def test_async_client_cache_does_not_reuse_clients_between_collected_loops() -> None: + cache: Final = LLMClientCache() + results: Final = tuple(_cache_result_on_new_loop(cache) for _ in range(50)) + loop_ids: Final = tuple(loop_id for loop_id, _ in results) + + assert all(result is None for _, result in results) + assert len(set(loop_ids)) < len(loop_ids) + + @pytest.mark.asyncio async def test_remove_key_does_not_close_async_client(): """ From 62a3096e24ea81faa8e5295e51b3fc436b6cf987 Mon Sep 17 00:00:00 2001 From: mateo Date: Fri, 16 Oct 2026 22:39:52 +0000 Subject: [PATCH 2/2] fix(caching): drop deferred closes queued on a collected event loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/evicted_client_closer.py | 41 +++++++- .../caching/test_evicted_client_closer.py | 96 +++++++++++-------- .../unit/caching/test_llm_caching_handler.py | 31 ++++-- 3 files changed, 117 insertions(+), 51 deletions(-) diff --git a/litellm/caching/evicted_client_closer.py b/litellm/caching/evicted_client_closer.py index bfd1edcccf9..523fbe106f7 100644 --- a/litellm/caching/evicted_client_closer.py +++ b/litellm/caching/evicted_client_closer.py @@ -29,6 +29,8 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to close them, and each bucket is ordered by deadline, so a reap walks the entries that are due rather than the whole queue. +A bucket whose loop has been collected is dropped, since nothing can await its closes any more. + The queue holds its clients weakly, so waiting out a grace window never keeps alive anything the collector would have reclaimed first. """ @@ -179,6 +181,8 @@ class EvictedClientCloser: self._clock = clock self._owned: weakref.WeakSet[object] = weakref.WeakSet() self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues + self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock + self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded self._pending_count = 0 self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes @@ -209,13 +213,19 @@ class EvictedClientCloser: close_fn: Final = _close_function(client) if close_fn is None: return + loop_token: Final = running_loop_token() + needs_loop: Final = inspect.iscoroutinefunction(close_fn) if self._pending_count >= self._max_pending: - return + self._discard_dead_loops() + if self._pending_count >= self._max_pending: + return + if needs_loop and loop_token is not None: + self._watch_loop(loop_token) self._enqueue( _PendingClose( client_ref=weakref.ref(client), - loop_token=running_loop_token(), - needs_loop=inspect.iscoroutinefunction(close_fn), + loop_token=loop_token, + needs_loop=needs_loop, close_after=self._clock() + self._grace_seconds, ) ) @@ -242,6 +252,29 @@ class EvictedClientCloser: def pending_count(self) -> int: return self._pending_count + def _watch_loop(self, loop_token: int) -> None: + if loop_token in self._watched_loop_tokens: + return + try: + weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token) + except TypeError: + return + self._watched_loop_tokens.add(loop_token) + + def _discard_dead_loops(self) -> None: + with self._queue_lock: + self._discard_dead_loops_locked() + + def _discard_dead_loops_locked(self) -> None: + while self._dead_loop_tokens: + self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft()) + + def _discard_dead_loop_token_locked(self, loop_token: int) -> None: + self._watched_loop_tokens.discard(loop_token) + bucket: Final = self._buckets.pop(loop_token, None) + if bucket is not None: + self._pending_count -= len(bucket) + def _enqueue(self, pending: _PendingClose) -> None: """Append to the entry's bucket, dropping any dead entries it queues behind. @@ -250,6 +283,7 @@ class EvictedClientCloser: the front rather than having to be searched for. """ with self._queue_lock: + self._discard_dead_loops_locked() bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque()) while bucket and bucket[0].client_ref() is None: bucket.popleft() @@ -262,6 +296,7 @@ class EvictedClientCloser: (_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token) ) with self._queue_lock: + self._discard_dead_loops_locked() return tuple(pending for key in buckets for pending in self._drain_locked(key, now)) def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]: diff --git a/tests/unit/caching/test_evicted_client_closer.py b/tests/unit/caching/test_evicted_client_closer.py index cfb6ded02fd..c0a2f292c7b 100644 --- a/tests/unit/caching/test_evicted_client_closer.py +++ b/tests/unit/caching/test_evicted_client_closer.py @@ -83,40 +83,6 @@ def make_closer(clock: FakeClock, grace_seconds: float = 60.0) -> EvictedClientC return EvictedClientCloser(grace_seconds=grace_seconds, clock=clock) -async def _schedule_client(closer: EvictedClientCloser, client: RecordingAsyncClient) -> None: - closer.schedule(client) - - -async def _reap_and_yield(closer: EvictedClientCloser) -> None: - closer.reap() - await asyncio.sleep(0) - - -def _reap_after_loop_id_reuse( - closer: EvictedClientCloser, - client: RecordingAsyncClient, - loop_ids: frozenset[int], - attempts: int, -) -> bool: - if attempts == 0: - return False - loop: Final = asyncio.new_event_loop() - loop_id: Final = id(loop) - if loop_id in loop_ids: - try: - loop.run_until_complete(_reap_and_yield(closer)) - finally: - loop.close() - return True - try: - loop.run_until_complete(_schedule_client(closer, client)) - finally: - loop.close() - del loop - gc.collect() - return _reap_after_loop_id_reuse(closer, client, loop_ids | {loop_id}, attempts - 1) - - @pytest.mark.asyncio async def test_redis_client_is_closed_only_after_its_subscription_releases_the_connection(): closer = EvictedClientCloser(grace_seconds=0) @@ -309,13 +275,12 @@ def test_sync_client_evicted_outside_an_event_loop_is_still_closed(): assert closer.pending_count == 0 -def test_async_client_from_collected_loop_is_not_closed_by_a_loop_reusing_its_id() -> None: +def test_async_client_from_collected_loop_is_not_closed_by_fresh_loops() -> None: clock: Final = FakeClock() closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock) client: Final = RecordingAsyncClient() closer.mark_owned(client) loop_a: Final = asyncio.new_event_loop() - loop_a_id: Final = id(loop_a) async def schedule() -> None: closer.schedule(client) @@ -326,14 +291,62 @@ def test_async_client_from_collected_loop_is_not_closed_by_a_loop_reusing_its_id loop_a.close() del loop_a gc.collect() - assert closer.pending_count == 1 - reused_loop_id: Final = _reap_after_loop_id_reuse(closer, client, frozenset({loop_a_id}), attempts=50) - assert reused_loop_id, "new event loops did not reuse a collected loop's id" + async def reap() -> None: + closer.reap() + await asyncio.sleep(0) + + def reap_on_new_loop() -> None: + loop: Final = asyncio.new_event_loop() + try: + loop.run_until_complete(reap()) + finally: + loop.close() + del loop + gc.collect() + + for _ in range(50): + reap_on_new_loop() assert client.close_calls == 0 +def test_closes_queued_on_a_collected_loop_free_capacity() -> None: + clock: Final = FakeClock() + closer: Final = EvictedClientCloser(grace_seconds=0, max_pending=1, clock=clock) + first: Final = RecordingAsyncClient() + closer.mark_owned(first) + loop_a: Final = asyncio.new_event_loop() + + async def schedule_first() -> None: + closer.schedule(first) + + try: + loop_a.run_until_complete(schedule_first()) + finally: + loop_a.close() + del loop_a + gc.collect() + + second: Final = RecordingAsyncClient() + closer.mark_owned(second) + loop_b: Final = asyncio.new_event_loop() + + async def schedule_and_reap_second() -> None: + closer.schedule(second) + closer.reap() + await asyncio.sleep(0) + + try: + loop_b.run_until_complete(schedule_and_reap_second()) + finally: + loop_b.close() + + assert first.close_calls == 0 + assert second.close_calls == 1 + assert closer.pending_count == 0 + + @pytest.mark.asyncio async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): clock = FakeClock() @@ -357,8 +370,7 @@ async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): @pytest.mark.asyncio -async def test_a_client_evicted_on_another_event_loop_is_left_alone(): - """Closing a client bound to a different loop would schedule work on that loop.""" +async def test_a_client_evicted_on_a_collected_loop_is_dropped(): clock = FakeClock() closer = make_closer(clock) client = AsyncClient() @@ -378,7 +390,7 @@ async def test_a_client_evicted_on_another_event_loop_is_left_alone(): await asyncio.sleep(0.05) assert client.closed is False - assert closer.pending_count == 1 + assert closer.pending_count == 0 @pytest.mark.asyncio diff --git a/tests/unit/caching/test_llm_caching_handler.py b/tests/unit/caching/test_llm_caching_handler.py index b0dc53baf11..92c9bf4d250 100644 --- a/tests/unit/caching/test_llm_caching_handler.py +++ b/tests/unit/caching/test_llm_caching_handler.py @@ -35,16 +35,30 @@ async def _get_and_set_cache(cache: LLMClientCache) -> object | None: return cached -def _cache_result_on_new_loop(cache: LLMClientCache) -> tuple[int, object | None]: +def _cache_result_on_new_loop(cache: LLMClientCache) -> object | None: loop: Final = asyncio.new_event_loop() - loop_id: Final = id(loop) try: result: Final = loop.run_until_complete(_get_and_set_cache(cache)) finally: loop.close() del loop gc.collect() - return loop_id, result + return result + + +async def _get_cache_key(cache: LLMClientCache) -> str: + return cache.update_cache_key_with_event_loop("client") + + +def _cache_key_on_new_loop(cache: LLMClientCache) -> str: + loop: Final = asyncio.new_event_loop() + try: + key: Final = loop.run_until_complete(_get_cache_key(cache)) + finally: + loop.close() + del loop + gc.collect() + return key class MockAsyncClient: @@ -70,10 +84,15 @@ class MockSyncClient: def test_async_client_cache_does_not_reuse_clients_between_collected_loops() -> None: cache: Final = LLMClientCache() results: Final = tuple(_cache_result_on_new_loop(cache) for _ in range(50)) - loop_ids: Final = tuple(loop_id for loop_id, _ in results) - assert all(result is None for _, result in results) - assert len(set(loop_ids)) < len(loop_ids) + assert all(result is None for result in results) + + +def test_async_client_cache_keys_differ_for_every_collected_loop() -> None: + cache: Final = LLMClientCache() + keys: Final = tuple(_cache_key_on_new_loop(cache) for _ in range(50)) + + assert len(set(keys)) == len(keys) @pytest.mark.asyncio