mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge e35b9206fe into 38b4d31447
This commit is contained in:
commit
0039d40ef4
2 changed files with 32 additions and 4 deletions
|
|
@ -389,7 +389,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
async with self._session_lock:
|
||||
if self._http_session is None or self._http_session.closed:
|
||||
self._http_session = aiohttp.ClientSession()
|
||||
yield self._http_session
|
||||
session: Final = self._http_session
|
||||
yield session
|
||||
else:
|
||||
# Background thread/loop -> use loop-bound session cache
|
||||
# This avoids "attached to a different loop" or "no running event loop" errors
|
||||
|
|
@ -638,9 +639,8 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
|
|||
# The semaphore is shared per event loop across every chunked call on
|
||||
# this instance, so many oversized blocks in one request (or many
|
||||
# concurrent requests) still hold at most this many analyzer calls in
|
||||
# flight. On the proxy's main thread the shared-session lock in
|
||||
# _get_session_iterator additionally serializes the HTTP calls; the
|
||||
# bound matters for loop-bound sessions (background threads).
|
||||
# flight. This bound applies to both the shared session and the
|
||||
# loop-bound sessions used by background threads.
|
||||
analyze_semaphore: Final = self._get_chunk_semaphore()
|
||||
|
||||
async def _analyze_chunk_bounded(
|
||||
|
|
|
|||
|
|
@ -1453,6 +1453,34 @@ async def test_get_session_iterator_thread_safety(presidio_guardrail):
|
|||
print("✓ Session iterator thread safety test passed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_session_iterator_lock_is_not_held_while_request_uses_session():
|
||||
"""
|
||||
The shared session lock must only protect session creation.
|
||||
|
||||
If it is held while callers use the session, every Presidio HTTP request in
|
||||
a proxy process is serialized even though async_pre_call_hook fans out with
|
||||
asyncio.gather.
|
||||
"""
|
||||
guardrail = OPTIONAL_PresidioPIIMasking(mock_testing=True)
|
||||
|
||||
entered = 0
|
||||
both_entered = asyncio.Event()
|
||||
|
||||
async def use_session():
|
||||
nonlocal entered
|
||||
async with guardrail._get_session_iterator() as session:
|
||||
assert session is guardrail._http_session
|
||||
entered += 1
|
||||
if entered == 2:
|
||||
both_entered.set()
|
||||
await asyncio.wait_for(both_entered.wait(), timeout=1.0)
|
||||
|
||||
await asyncio.gather(use_session(), use_session())
|
||||
|
||||
await guardrail._close_http_session()
|
||||
|
||||
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue