mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge 62a3096e24 into e04e89c701
This commit is contained in:
commit
c698ad7a0d
4 changed files with 194 additions and 23 deletions
|
|
@ -29,6 +29,8 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
|
|||
close them, and each bucket is ordered by deadline, so a reap walks the entries
|
||||
that are due rather than the whole queue.
|
||||
|
||||
A bucket whose loop has been collected is dropped, since nothing can await its closes any more.
|
||||
|
||||
The queue holds its clients weakly, so waiting out a grace window never keeps
|
||||
alive anything the collector would have reclaimed first.
|
||||
"""
|
||||
|
|
@ -36,6 +38,7 @@ alive anything the collector would have reclaimed first.
|
|||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import itertools
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
|
|
@ -53,6 +56,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
|
|||
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
|
||||
|
||||
_BucketKey = str | int
|
||||
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
|
||||
_LOOP_TOKEN_COUNTER: Final = itertools.count()
|
||||
_LOOP_TOKEN_LOCK: Final = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -63,12 +69,12 @@ class _PendingClose:
|
|||
collector would otherwise have reclaimed first.
|
||||
|
||||
``needs_loop`` is set for a client whose close is a coroutine; those can only
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_id``.
|
||||
be closed from the event loop they were evicted on, recorded in ``loop_token``.
|
||||
A client that closes synchronously carries neither constraint.
|
||||
"""
|
||||
|
||||
client_ref: "weakref.ref[object]"
|
||||
loop_id: int | None
|
||||
loop_token: int | None
|
||||
needs_loop: bool
|
||||
close_after: float
|
||||
|
||||
|
|
@ -77,16 +83,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
|
|||
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
|
||||
if not pending.needs_loop:
|
||||
return _CLOSABLE_ANYWHERE
|
||||
if pending.loop_id is None:
|
||||
if pending.loop_token is None:
|
||||
return _CLOSABLE_ON_ANY_LOOP
|
||||
return pending.loop_id
|
||||
return pending.loop_token
|
||||
|
||||
|
||||
def _running_loop_id() -> int | None:
|
||||
def running_loop_token() -> int | None:
|
||||
try:
|
||||
return id(asyncio.get_running_loop())
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return None
|
||||
try:
|
||||
token: Final[int | None] = _LOOP_TOKENS.get(loop)
|
||||
if token is not None:
|
||||
return token
|
||||
with _LOOP_TOKEN_LOCK:
|
||||
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
|
||||
except TypeError:
|
||||
return id(loop)
|
||||
|
||||
|
||||
def _close_function(client: object) -> Callable[[], object] | None:
|
||||
|
|
@ -167,6 +181,8 @@ class EvictedClientCloser:
|
|||
self._clock = clock
|
||||
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
|
||||
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
|
||||
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
|
||||
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
|
||||
self._pending_count = 0
|
||||
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
|
||||
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
|
||||
|
|
@ -197,13 +213,19 @@ class EvictedClientCloser:
|
|||
close_fn: Final = _close_function(client)
|
||||
if close_fn is None:
|
||||
return
|
||||
loop_token: Final = running_loop_token()
|
||||
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
self._discard_dead_loops()
|
||||
if self._pending_count >= self._max_pending:
|
||||
return
|
||||
if needs_loop and loop_token is not None:
|
||||
self._watch_loop(loop_token)
|
||||
self._enqueue(
|
||||
_PendingClose(
|
||||
client_ref=weakref.ref(client),
|
||||
loop_id=_running_loop_id(),
|
||||
needs_loop=inspect.iscoroutinefunction(close_fn),
|
||||
loop_token=loop_token,
|
||||
needs_loop=needs_loop,
|
||||
close_after=self._clock() + self._grace_seconds,
|
||||
)
|
||||
)
|
||||
|
|
@ -217,7 +239,7 @@ class EvictedClientCloser:
|
|||
if not self._pending_count:
|
||||
return
|
||||
now: Final = self._clock()
|
||||
for pending in self._take_due(_running_loop_id(), now):
|
||||
for pending in self._take_due(running_loop_token(), now):
|
||||
client = pending.client_ref()
|
||||
if client is None:
|
||||
continue
|
||||
|
|
@ -230,6 +252,29 @@ class EvictedClientCloser:
|
|||
def pending_count(self) -> int:
|
||||
return self._pending_count
|
||||
|
||||
def _watch_loop(self, loop_token: int) -> None:
|
||||
if loop_token in self._watched_loop_tokens:
|
||||
return
|
||||
try:
|
||||
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
|
||||
except TypeError:
|
||||
return
|
||||
self._watched_loop_tokens.add(loop_token)
|
||||
|
||||
def _discard_dead_loops(self) -> None:
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
|
||||
def _discard_dead_loops_locked(self) -> None:
|
||||
while self._dead_loop_tokens:
|
||||
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
|
||||
|
||||
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
|
||||
self._watched_loop_tokens.discard(loop_token)
|
||||
bucket: Final = self._buckets.pop(loop_token, None)
|
||||
if bucket is not None:
|
||||
self._pending_count -= len(bucket)
|
||||
|
||||
def _enqueue(self, pending: _PendingClose) -> None:
|
||||
"""Append to the entry's bucket, dropping any dead entries it queues behind.
|
||||
|
||||
|
|
@ -238,6 +283,7 @@ class EvictedClientCloser:
|
|||
the front rather than having to be searched for.
|
||||
"""
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
|
||||
while bucket and bucket[0].client_ref() is None:
|
||||
bucket.popleft()
|
||||
|
|
@ -245,9 +291,12 @@ class EvictedClientCloser:
|
|||
bucket.append(pending)
|
||||
self._pending_count += 1
|
||||
|
||||
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
|
||||
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
|
||||
buckets = (
|
||||
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
|
||||
)
|
||||
with self._queue_lock:
|
||||
self._discard_dead_loops_locked()
|
||||
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
|
||||
|
||||
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@
|
|||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import Final
|
||||
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
|
||||
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
|
||||
from .in_memory_cache import InMemoryCache
|
||||
|
||||
|
||||
|
|
@ -48,12 +47,8 @@ class LLMClientCache(InMemoryCache):
|
|||
Add the event loop to the cache key, to prevent event loop closed errors.
|
||||
If none, use the key as is.
|
||||
"""
|
||||
try:
|
||||
event_loop: Final = asyncio.get_running_loop()
|
||||
stringified_event_loop: Final = str(id(event_loop))
|
||||
return f"{key}-{stringified_event_loop}"
|
||||
except RuntimeError: # handle no current running event loop
|
||||
return key
|
||||
token: Final = running_loop_token()
|
||||
return f"{key}-{token}" if token is not None else key
|
||||
|
||||
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
|
||||
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ never closed, because litellm does not own its lifecycle.
|
|||
import asyncio
|
||||
import gc
|
||||
import weakref
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
|
|
@ -41,6 +42,14 @@ class AsyncClient:
|
|||
self.closed = True
|
||||
|
||||
|
||||
class RecordingAsyncClient:
|
||||
def __init__(self) -> None:
|
||||
self.close_calls = 0
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self.close_calls += 1
|
||||
|
||||
|
||||
class SyncClient:
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
|
|
@ -266,6 +275,78 @@ def test_sync_client_evicted_outside_an_event_loop_is_still_closed():
|
|||
assert closer.pending_count == 0
|
||||
|
||||
|
||||
def test_async_client_from_collected_loop_is_not_closed_by_fresh_loops() -> None:
|
||||
clock: Final = FakeClock()
|
||||
closer: Final = EvictedClientCloser(grace_seconds=0, clock=clock)
|
||||
client: Final = RecordingAsyncClient()
|
||||
closer.mark_owned(client)
|
||||
loop_a: Final = asyncio.new_event_loop()
|
||||
|
||||
async def schedule() -> None:
|
||||
closer.schedule(client)
|
||||
|
||||
try:
|
||||
loop_a.run_until_complete(schedule())
|
||||
finally:
|
||||
loop_a.close()
|
||||
del loop_a
|
||||
gc.collect()
|
||||
|
||||
async def reap() -> None:
|
||||
closer.reap()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
def reap_on_new_loop() -> None:
|
||||
loop: Final = asyncio.new_event_loop()
|
||||
try:
|
||||
loop.run_until_complete(reap())
|
||||
finally:
|
||||
loop.close()
|
||||
del loop
|
||||
gc.collect()
|
||||
|
||||
for _ in range(50):
|
||||
reap_on_new_loop()
|
||||
|
||||
assert client.close_calls == 0
|
||||
|
||||
|
||||
def test_closes_queued_on_a_collected_loop_free_capacity() -> None:
|
||||
clock: Final = FakeClock()
|
||||
closer: Final = EvictedClientCloser(grace_seconds=0, max_pending=1, clock=clock)
|
||||
first: Final = RecordingAsyncClient()
|
||||
closer.mark_owned(first)
|
||||
loop_a: Final = asyncio.new_event_loop()
|
||||
|
||||
async def schedule_first() -> None:
|
||||
closer.schedule(first)
|
||||
|
||||
try:
|
||||
loop_a.run_until_complete(schedule_first())
|
||||
finally:
|
||||
loop_a.close()
|
||||
del loop_a
|
||||
gc.collect()
|
||||
|
||||
second: Final = RecordingAsyncClient()
|
||||
closer.mark_owned(second)
|
||||
loop_b: Final = asyncio.new_event_loop()
|
||||
|
||||
async def schedule_and_reap_second() -> None:
|
||||
closer.schedule(second)
|
||||
closer.reap()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
try:
|
||||
loop_b.run_until_complete(schedule_and_reap_second())
|
||||
finally:
|
||||
loop_b.close()
|
||||
|
||||
assert first.close_calls == 0
|
||||
assert second.close_calls == 1
|
||||
assert closer.pending_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped():
|
||||
clock = FakeClock()
|
||||
|
|
@ -289,8 +370,7 @@ async def test_an_async_client_waits_for_a_loop_rather_than_being_dropped():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_client_evicted_on_another_event_loop_is_left_alone():
|
||||
"""Closing a client bound to a different loop would schedule work on that loop."""
|
||||
async def test_a_client_evicted_on_a_collected_loop_is_dropped():
|
||||
clock = FakeClock()
|
||||
closer = make_closer(clock)
|
||||
client = AsyncClient()
|
||||
|
|
@ -310,7 +390,7 @@ async def test_a_client_evicted_on_another_event_loop_is_left_alone():
|
|||
await asyncio.sleep(0.05)
|
||||
|
||||
assert client.closed is False
|
||||
assert closer.pending_count == 1
|
||||
assert closer.pending_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ See: https://github.com/BerriAI/litellm/pull/22247
|
|||
|
||||
import asyncio, gc, httpx, importlib, litellm, os, threading, time, weakref
|
||||
import warnings
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -28,6 +29,38 @@ from litellm.utils import _invalidate_model_cost_lowercase_map
|
|||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
||||
|
||||
async def _get_and_set_cache(cache: LLMClientCache) -> object | None:
|
||||
cached: Final = await cache.async_get_cache("client")
|
||||
await cache.async_set_cache("client", object())
|
||||
return cached
|
||||
|
||||
|
||||
def _cache_result_on_new_loop(cache: LLMClientCache) -> object | None:
|
||||
loop: Final = asyncio.new_event_loop()
|
||||
try:
|
||||
result: Final = loop.run_until_complete(_get_and_set_cache(cache))
|
||||
finally:
|
||||
loop.close()
|
||||
del loop
|
||||
gc.collect()
|
||||
return result
|
||||
|
||||
|
||||
async def _get_cache_key(cache: LLMClientCache) -> str:
|
||||
return cache.update_cache_key_with_event_loop("client")
|
||||
|
||||
|
||||
def _cache_key_on_new_loop(cache: LLMClientCache) -> str:
|
||||
loop: Final = asyncio.new_event_loop()
|
||||
try:
|
||||
key: Final = loop.run_until_complete(_get_cache_key(cache))
|
||||
finally:
|
||||
loop.close()
|
||||
del loop
|
||||
gc.collect()
|
||||
return key
|
||||
|
||||
|
||||
class MockAsyncClient:
|
||||
"""Mock async HTTP client with an async close method."""
|
||||
|
||||
|
|
@ -48,6 +81,20 @@ class MockSyncClient:
|
|||
self.closed = True
|
||||
|
||||
|
||||
def test_async_client_cache_does_not_reuse_clients_between_collected_loops() -> None:
|
||||
cache: Final = LLMClientCache()
|
||||
results: Final = tuple(_cache_result_on_new_loop(cache) for _ in range(50))
|
||||
|
||||
assert all(result is None for result in results)
|
||||
|
||||
|
||||
def test_async_client_cache_keys_differ_for_every_collected_loop() -> None:
|
||||
cache: Final = LLMClientCache()
|
||||
keys: Final = tuple(_cache_key_on_new_loop(cache) for _ in range(50))
|
||||
|
||||
assert len(set(keys)) == len(keys)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remove_key_does_not_close_async_client():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue