This commit is contained in:
Mateo Wang 2026-10-24 08:30:28 +00:00 • committed by GitHub
commit c698ad7a0d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 194 additions and 23 deletions

View file

@ -29,6 +29,8 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
close them, and each bucket is ordered by deadline, so a reap walks the entries
that are due rather than the whole queue.
A bucket whose loop has been collected is dropped, since nothing can await its closes any more.
The queue holds its clients weakly, so waiting out a grace window never keeps
alive anything the collector would have reclaimed first.
"""
@ -36,6 +38,7 @@ alive anything the collector would have reclaimed first.
import asyncio
import contextlib
import inspect
import itertools
import threading
import time
import weakref
@ -53,6 +56,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
_BucketKey = str | int
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
_LOOP_TOKEN_COUNTER: Final = itertools.count()
_LOOP_TOKEN_LOCK: Final = threading.Lock()
@dataclass(frozen=True, slots=True)
@ -63,12 +69,12 @@ class _PendingClose:
collector would otherwise have reclaimed first.
``needs_loop`` is set for a client whose close is a coroutine; those can only
be closed from the event loop they were evicted on, recorded in ``loop_id``.
be closed from the event loop they were evicted on, recorded in ``loop_token``.
A client that closes synchronously carries neither constraint.
"""
client_ref: "weakref.ref[object]"
loop_id: int | None
loop_token: int | None
needs_loop: bool
close_after: float
@ -77,16 +83,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
if not pending.needs_loop:
return _CLOSABLE_ANYWHERE
if pending.loop_id is None:
if pending.loop_token is None:
return _CLOSABLE_ON_ANY_LOOP
return pending.loop_id
return pending.loop_token
def _running_loop_id() -> int | None:
def running_loop_token() -> int | None:
try:
return id(asyncio.get_running_loop())
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return None
try:
token: Final[int | None] = _LOOP_TOKENS.get(loop)
if token is not None:
return token
with _LOOP_TOKEN_LOCK:
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
except TypeError:
return id(loop)
def _close_function(client: object) -> Callable[[], object] | None:
@ -167,6 +181,8 @@ class EvictedClientCloser:
self._clock = clock
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
self._pending_count = 0
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
@ -197,13 +213,19 @@ class EvictedClientCloser:
close_fn: Final = _close_function(client)
if close_fn is None:
return
loop_token: Final = running_loop_token()
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
if self._pending_count >= self._max_pending:
return
self._discard_dead_loops()
if self._pending_count >= self._max_pending:
return
if needs_loop and loop_token is not None:
self._watch_loop(loop_token)
self._enqueue(
_PendingClose(
client_ref=weakref.ref(client),
loop_id=_running_loop_id(),
needs_loop=inspect.iscoroutinefunction(close_fn),
loop_token=loop_token,
needs_loop=needs_loop,
close_after=self._clock() + self._grace_seconds,
)
)
@ -217,7 +239,7 @@ class EvictedClientCloser:
if not self._pending_count:
return
now: Final = self._clock()
for pending in self._take_due(_running_loop_id(), now):
for pending in self._take_due(running_loop_token(), now):
client = pending.client_ref()
if client is None:
continue
@ -230,6 +252,29 @@ class EvictedClientCloser:
def pending_count(self) -> int:
return self._pending_count
def _watch_loop(self, loop_token: int) -> None:
if loop_token in self._watched_loop_tokens:
return
try:
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
except TypeError:
return
self._watched_loop_tokens.add(loop_token)
def _discard_dead_loops(self) -> None:
with self._queue_lock:
self._discard_dead_loops_locked()
def _discard_dead_loops_locked(self) -> None:
while self._dead_loop_tokens:
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
self._watched_loop_tokens.discard(loop_token)
bucket: Final = self._buckets.pop(loop_token, None)
if bucket is not None:
self._pending_count -= len(bucket)
def _enqueue(self, pending: _PendingClose) -> None:
"""Append to the entry's bucket, dropping any dead entries it queues behind.
@ -238,6 +283,7 @@ class EvictedClientCloser:
the front rather than having to be searched for.
"""
with self._queue_lock:
self._discard_dead_loops_locked()
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
while bucket and bucket[0].client_ref() is None:
bucket.popleft()
@ -245,9 +291,12 @@ class EvictedClientCloser:
bucket.append(pending)
self._pending_count += 1
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
)
with self._queue_lock:
self._discard_dead_loops_locked()
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:

View file

@ -2,10 +2,9 @@
Add the event loop to the cache key, to prevent event loop closed errors.
"""
import asyncio
from typing import Final
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
from .in_memory_cache import InMemoryCache
@ -48,12 +47,8 @@ class LLMClientCache(InMemoryCache):
Add the event loop to the cache key, to prevent event loop closed errors.
If none, use the key as is.
"""
try:
event_loop: Final = asyncio.get_running_loop()
stringified_event_loop: Final = str(id(event_loop))
return f"{key}-{stringified_event_loop}"
except RuntimeError: # handle no current running event loop
return key
token: Final = running_loop_token()
return f"{key}-{token}" if token is not None else key
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""

View file

@ -10,6 +10,7 @@ never closed, because litellm does not own its lifecycle.
import asyncio
import gc
import weakref
from typing import Final
from unittest.mock import AsyncMock
import httpx
@ -41,6 +42,14 @@ class AsyncClient:
self.closed = True
class RecordingAsyncClient:
def __init__(self) -> None:
self.close_calls = 0
async def aclose(self) -> None:
self.close_calls += 1
class SyncClient:
def __init__(self) -> None:
self.closed = False
@ -266,6 +275,78 @@ def test_sync_client_evicted_outside_an_event_loop_is_still_closed():
assert closer.pending_count == 0
def test_async_client_from_collected_loop_is_not_closed_by_fresh_loops() -> None:
clock: Final = FakeClock()
closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock)
client: Final = RecordingAsyncClient()
closer.mark_owned(client)
loop_a: Final = asyncio.new_event_loop()
async def schedule() -> None:
closer.schedule(client)
try:
loop_a.run_until_complete(schedule())
finally:
loop_a.close()
del loop_a
gc.collect()
async def reap() -> None:
closer.reap()
await asyncio.sleep(0)
def reap_on_new_loop() -> None:
loop: Final = asyncio.new_event_loop()
try:
loop.run_until_complete(reap())
finally:
loop.close()
del loop
gc.collect()
for _ in range(50):
reap_on_new_loop()
assert client.close_calls == 0
def test_closes_queued_on_a_collected_loop_free_capacity() -> None:
clock: Final = FakeClock()
closer: Final = EvictedClientCloser(grace_seconds=0, max_pending=1, clock=clock)
first: Final = RecordingAsyncClient()
closer.mark_owned(first)
loop_a: Final = asyncio.new_event_loop()
async def schedule_first() -> None:
closer.schedule(first)
try:
loop_a.run_until_complete(schedule_first())
finally:
loop_a.close()
del loop_a
gc.collect()
second: Final = RecordingAsyncClient()
closer.mark_owned(second)
loop_b: Final = asyncio.new_event_loop()
async def schedule_and_reap_second() -> None:
closer.schedule(second)
closer.reap()
await asyncio.sleep(0)
try:
loop_b.run_until_complete(schedule_and_reap_second())
finally:
loop_b.close()
assert first.close_calls == 0
assert second.close_calls == 1
assert closer.pending_count == 0
@pytest.mark.asyncio
async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped():
clock = FakeClock()
@ -289,8 +370,7 @@ async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped():
@pytest.mark.asyncio
async def test_a_client_evicted_on_another_event_loop_is_left_alone():
"""Closing a client bound to a different loop would schedule work on that loop."""
async def test_a_client_evicted_on_a_collected_loop_is_dropped():
clock = FakeClock()
closer = make_closer(clock)
client = AsyncClient()
@ -310,7 +390,7 @@ async def test_a_client_evicted_on_another_event_loop_is_left_alone():
await asyncio.sleep(0.05)
assert client.closed is False
assert closer.pending_count == 1
assert closer.pending_count == 0
@pytest.mark.asyncio

View file

@ -10,6 +10,7 @@ See: https://github.com/BerriAI/litellm/pull/22247
import asyncio, gc, httpx, importlib, litellm, os, threading, time, weakref
import warnings
from typing import Final
import pytest
@ -28,6 +29,38 @@ from litellm.utils import _invalidate_model_cost_lowercase_map
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
async def _get_and_set_cache(cache: LLMClientCache) -> object | None:
cached: Final = await cache.async_get_cache("client")
await cache.async_set_cache("client", object())
return cached
def _cache_result_on_new_loop(cache: LLMClientCache) -> object | None:
loop: Final = asyncio.new_event_loop()
try:
result: Final = loop.run_until_complete(_get_and_set_cache(cache))
finally:
loop.close()
del loop
gc.collect()
return result
async def _get_cache_key(cache: LLMClientCache) -> str:
return cache.update_cache_key_with_event_loop("client")
def _cache_key_on_new_loop(cache: LLMClientCache) -> str:
loop: Final = asyncio.new_event_loop()
try:
key: Final = loop.run_until_complete(_get_cache_key(cache))
finally:
loop.close()
del loop
gc.collect()
return key
class MockAsyncClient:
"""Mock async HTTP client with an async close method."""
@ -48,6 +81,20 @@ class MockSyncClient:
self.closed = True
def test_async_client_cache_does_not_reuse_clients_between_collected_loops() -> None:
cache: Final = LLMClientCache()
results: Final = tuple(_cache_result_on_new_loop(cache) for _ in range(50))
assert all(result is None for result in results)
def test_async_client_cache_keys_differ_for_every_collected_loop() -> None:
cache: Final = LLMClientCache()
keys: Final = tuple(_cache_key_on_new_loop(cache) for _ in range(50))
assert len(set(keys)) == len(keys)
@pytest.mark.asyncio
async def test_remove_key_does_not_close_async_client():
"""