diff --git a/litellm/caching/evicted_client_closer.py b/litellm/caching/evicted_client_closer.py index b22cd3aecd1..523fbe106f7 100644 --- a/litellm/caching/evicted_client_closer.py +++ b/litellm/caching/evicted_client_closer.py @@ -29,6 +29,8 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to close them, and each bucket is ordered by deadline, so a reap walks the entries that are due rather than the whole queue. +A bucket whose loop has been collected is dropped, since nothing can await its closes any more. + The queue holds its clients weakly, so waiting out a grace window never keeps alive anything the collector would have reclaimed first. """ @@ -36,6 +38,7 @@ alive anything the collector would have reclaimed first. import asyncio import contextlib import inspect +import itertools import threading import time import weakref @@ -53,6 +56,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere" _CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop" _BucketKey = str | int +_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary() +_LOOP_TOKEN_COUNTER: Final = itertools.count() +_LOOP_TOKEN_LOCK: Final = threading.Lock() @dataclass(frozen=True, slots=True) @@ -63,12 +69,12 @@ class _PendingClose: collector would otherwise have reclaimed first. ``needs_loop`` is set for a client whose close is a coroutine; those can only - be closed from the event loop they were evicted on, recorded in ``loop_id``. + be closed from the event loop they were evicted on, recorded in ``loop_token``. A client that closes synchronously carries neither constraint. """ client_ref: "weakref.ref[object]" - loop_id: int | None + loop_token: int | None needs_loop: bool close_after: float @@ -77,16 +83,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey: """Which reaps can close this entry: any at all, any running a loop, or one loop's.""" if not pending.needs_loop: return _CLOSABLE_ANYWHERE - if pending.loop_id is None: + if pending.loop_token is None: return _CLOSABLE_ON_ANY_LOOP - return pending.loop_id + return pending.loop_token -def _running_loop_id() -> int | None: +def running_loop_token() -> int | None: try: - return id(asyncio.get_running_loop()) + loop: Final = asyncio.get_running_loop() except RuntimeError: return None + try: + token: Final[int | None] = _LOOP_TOKENS.get(loop) + if token is not None: + return token + with _LOOP_TOKEN_LOCK: + return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER)) + except TypeError: + return id(loop) def _close_function(client: object) -> Callable[[], object] | None: @@ -167,6 +181,8 @@ class EvictedClientCloser: self._clock = clock self._owned: weakref.WeakSet[object] = weakref.WeakSet() self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues + self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock + self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded self._pending_count = 0 self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes @@ -197,13 +213,19 @@ class EvictedClientCloser: close_fn: Final = _close_function(client) if close_fn is None: return + loop_token: Final = running_loop_token() + needs_loop: Final = inspect.iscoroutinefunction(close_fn) if self._pending_count >= self._max_pending: - return + self._discard_dead_loops() + if self._pending_count >= self._max_pending: + return + if needs_loop and loop_token is not None: + self._watch_loop(loop_token) self._enqueue( _PendingClose( client_ref=weakref.ref(client), - loop_id=_running_loop_id(), - needs_loop=inspect.iscoroutinefunction(close_fn), + loop_token=loop_token, + needs_loop=needs_loop, close_after=self._clock() + self._grace_seconds, ) ) @@ -217,7 +239,7 @@ class EvictedClientCloser: if not self._pending_count: return now: Final = self._clock() - for pending in self._take_due(_running_loop_id(), now): + for pending in self._take_due(running_loop_token(), now): client = pending.client_ref() if client is None: continue @@ -230,6 +252,29 @@ class EvictedClientCloser: def pending_count(self) -> int: return self._pending_count + def _watch_loop(self, loop_token: int) -> None: + if loop_token in self._watched_loop_tokens: + return + try: + weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token) + except TypeError: + return + self._watched_loop_tokens.add(loop_token) + + def _discard_dead_loops(self) -> None: + with self._queue_lock: + self._discard_dead_loops_locked() + + def _discard_dead_loops_locked(self) -> None: + while self._dead_loop_tokens: + self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft()) + + def _discard_dead_loop_token_locked(self, loop_token: int) -> None: + self._watched_loop_tokens.discard(loop_token) + bucket: Final = self._buckets.pop(loop_token, None) + if bucket is not None: + self._pending_count -= len(bucket) + def _enqueue(self, pending: _PendingClose) -> None: """Append to the entry's bucket, dropping any dead entries it queues behind. @@ -238,6 +283,7 @@ class EvictedClientCloser: the front rather than having to be searched for. """ with self._queue_lock: + self._discard_dead_loops_locked() bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque()) while bucket and bucket[0].client_ref() is None: bucket.popleft() @@ -245,9 +291,12 @@ class EvictedClientCloser: bucket.append(pending) self._pending_count += 1 - def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]: - buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id) + def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]: + buckets = ( + (_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token) + ) with self._queue_lock: + self._discard_dead_loops_locked() return tuple(pending for key in buckets for pending in self._drain_locked(key, now)) def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]: diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index a89e43b78b4..beb9301e570 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -2,10 +2,9 @@ Add the event loop to the cache key, to prevent event loop closed errors. """ -import asyncio from typing import Final -from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer +from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token from .in_memory_cache import InMemoryCache @@ -48,12 +47,8 @@ class LLMClientCache(InMemoryCache): Add the event loop to the cache key, to prevent event loop closed errors. If none, use the key as is. """ - try: - event_loop: Final = asyncio.get_running_loop() - stringified_event_loop: Final = str(id(event_loop)) - return f"{key}-{stringified_event_loop}" - except RuntimeError: # handle no current running event loop - return key + token: Final = running_loop_token() + return f"{key}-{token}" if token is not None else key def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs): """``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted.""" diff --git a/tests/unit/caching/test_evicted_client_closer.py b/tests/unit/caching/test_evicted_client_closer.py index 7679e276621..c0a2f292c7b 100644 --- a/tests/unit/caching/test_evicted_client_closer.py +++ b/tests/unit/caching/test_evicted_client_closer.py @@ -10,6 +10,7 @@ never closed, because litellm does not own its lifecycle. import asyncio import gc import weakref +from typing import Final from unittest.mock import AsyncMock import httpx @@ -41,6 +42,14 @@ class AsyncClient: self.closed = True +class RecordingAsyncClient: + def __init__(self) -> None: + self.close_calls = 0 + + async def aclose(self) -> None: + self.close_calls += 1 + + class SyncClient: def __init__(self) -> None: self.closed = False @@ -266,6 +275,78 @@ def test_sync_client_evicted_outside_an_event_loop_is_still_closed(): assert closer.pending_count == 0 +def test_async_client_from_collected_loop_is_not_closed_by_fresh_loops() -> None: + clock: Final = FakeClock() + closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock) + client: Final = RecordingAsyncClient() + closer.mark_owned(client) + loop_a: Final = asyncio.new_event_loop() + + async def schedule() -> None: + closer.schedule(client) + + try: + loop_a.run_until_complete(schedule()) + finally: + loop_a.close() + del loop_a + gc.collect() + + async def reap() -> None: + closer.reap() + await asyncio.sleep(0) + + def reap_on_new_loop() -> None: + loop: Final = asyncio.new_event_loop() + try: + loop.run_until_complete(reap()) + finally: + loop.close() + del loop + gc.collect() + + for _ in range(50): + reap_on_new_loop() + + assert client.close_calls == 0 + + +def test_closes_queued_on_a_collected_loop_free_capacity() -> None: + clock: Final = FakeClock() + closer: Final = EvictedClientCloser(grace_seconds=0, max_pending=1, clock=clock) + first: Final = RecordingAsyncClient() + closer.mark_owned(first) + loop_a: Final = asyncio.new_event_loop() + + async def schedule_first() -> None: + closer.schedule(first) + + try: + loop_a.run_until_complete(schedule_first()) + finally: + loop_a.close() + del loop_a + gc.collect() + + second: Final = RecordingAsyncClient() + closer.mark_owned(second) + loop_b: Final = asyncio.new_event_loop() + + async def schedule_and_reap_second() -> None: + closer.schedule(second) + closer.reap() + await asyncio.sleep(0) + + try: + loop_b.run_until_complete(schedule_and_reap_second()) + finally: + loop_b.close() + + assert first.close_calls == 0 + assert second.close_calls == 1 + assert closer.pending_count == 0 + + @pytest.mark.asyncio async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): clock = FakeClock() @@ -289,8 +370,7 @@ async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped(): @pytest.mark.asyncio -async def test_a_client_evicted_on_another_event_loop_is_left_alone(): - """Closing a client bound to a different loop would schedule work on that loop.""" +async def test_a_client_evicted_on_a_collected_loop_is_dropped(): clock = FakeClock() closer = make_closer(clock) client = AsyncClient() @@ -310,7 +390,7 @@ async def test_a_client_evicted_on_another_event_loop_is_left_alone(): await asyncio.sleep(0.05) assert client.closed is False - assert closer.pending_count == 1 + assert closer.pending_count == 0 @pytest.mark.asyncio diff --git a/tests/unit/caching/test_llm_caching_handler.py b/tests/unit/caching/test_llm_caching_handler.py index 0c0fe8ca3c5..92c9bf4d250 100644 --- a/tests/unit/caching/test_llm_caching_handler.py +++ b/tests/unit/caching/test_llm_caching_handler.py @@ -10,6 +10,7 @@ See: https://github.com/BerriAI/litellm/pull/22247 import asyncio, gc, httpx, importlib, litellm, os, threading, time, weakref import warnings +from typing import Final import pytest @@ -28,6 +29,38 @@ from litellm.utils import _invalidate_model_cost_lowercase_map from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome +async def _get_and_set_cache(cache: LLMClientCache) -> object | None: + cached: Final = await cache.async_get_cache("client") + await cache.async_set_cache("client", object()) + return cached + + +def _cache_result_on_new_loop(cache: LLMClientCache) -> object | None: + loop: Final = asyncio.new_event_loop() + try: + result: Final = loop.run_until_complete(_get_and_set_cache(cache)) + finally: + loop.close() + del loop + gc.collect() + return result + + +async def _get_cache_key(cache: LLMClientCache) -> str: + return cache.update_cache_key_with_event_loop("client") + + +def _cache_key_on_new_loop(cache: LLMClientCache) -> str: + loop: Final = asyncio.new_event_loop() + try: + key: Final = loop.run_until_complete(_get_cache_key(cache)) + finally: + loop.close() + del loop + gc.collect() + return key + + class MockAsyncClient: """Mock async HTTP client with an async close method.""" @@ -48,6 +81,20 @@ class MockSyncClient: self.closed = True +def test_async_client_cache_does_not_reuse_clients_between_collected_loops() -> None: + cache: Final = LLMClientCache() + results: Final = tuple(_cache_result_on_new_loop(cache) for _ in range(50)) + + assert all(result is None for result in results) + + +def test_async_client_cache_keys_differ_for_every_collected_loop() -> None: + cache: Final = LLMClientCache() + keys: Final = tuple(_cache_key_on_new_loop(cache) for _ in range(50)) + + assert len(set(keys)) == len(keys) + + @pytest.mark.asyncio async def test_remove_key_does_not_close_async_client(): """