mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge dce51ce3cb into dddbaa803d
This commit is contained in:
commit
5106df6e07
2 changed files with 141 additions and 10 deletions
|
|
@ -633,6 +633,7 @@ class AsyncHTTPHandler:
|
|||
shared_session: Optional["ClientSession"] = None,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
follow_redirects: bool = True,
|
||||
client: httpx.AsyncClient | None = None,
|
||||
):
|
||||
self.timeout = timeout
|
||||
self.event_hooks = event_hooks
|
||||
|
|
@ -640,12 +641,16 @@ class AsyncHTTPHandler:
|
|||
self.shared_session = shared_session
|
||||
self.transport = transport
|
||||
self.follow_redirects = follow_redirects
|
||||
self._owns_client = True
|
||||
self._client = self.create_client(
|
||||
timeout=timeout,
|
||||
event_hooks=event_hooks,
|
||||
ssl_verify=ssl_verify,
|
||||
shared_session=shared_session,
|
||||
self._owns_client = client is None
|
||||
self._client = (
|
||||
client
|
||||
if client is not None
|
||||
else self.create_client(
|
||||
timeout=timeout,
|
||||
event_hooks=event_hooks,
|
||||
ssl_verify=ssl_verify,
|
||||
shared_session=shared_session,
|
||||
)
|
||||
)
|
||||
self.client_alias = client_alias
|
||||
|
||||
|
|
@ -1762,7 +1767,10 @@ def get_async_httpx_client(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
_cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider
|
||||
# An explicit transport (in-process ASGI, pinned egress) wins over the session; the key holds the session so a swap never serves a stale handler.
|
||||
session: Final = litellm.aclient_session if params is None or "transport" not in params else None
|
||||
_session_key_name: Final = f"_session{id(session)}" if session is not None else ""
|
||||
_cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider + _session_key_name
|
||||
|
||||
# Lazily initialize the global in-memory client cache to avoid relying on
|
||||
# litellm globals being fully populated during import time.
|
||||
|
|
@ -1781,11 +1789,12 @@ def get_async_httpx_client(
|
|||
# Filter out params that are only used for cache key, not for AsyncHTTPHandler.__init__
|
||||
handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"}
|
||||
handler_params["shared_session"] = shared_session
|
||||
_new_client = AsyncHTTPHandler(**handler_params)
|
||||
_new_client = AsyncHTTPHandler(**handler_params, client=session)
|
||||
else:
|
||||
_new_client = AsyncHTTPHandler(
|
||||
timeout=_default_cached_client_timeout(),
|
||||
shared_session=shared_session,
|
||||
client=session,
|
||||
)
|
||||
|
||||
cache.set_cache(
|
||||
|
|
@ -1812,7 +1821,9 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
_cache_key_name: Final = "httpx_client" + _params_key_name
|
||||
session: Final = litellm.client_session
|
||||
_session_key_name: Final = f"_session{id(session)}" if session is not None else ""
|
||||
_cache_key_name: Final = "httpx_client" + _params_key_name + _session_key_name
|
||||
|
||||
# Lazily initialize the global in-memory client cache to avoid relying on
|
||||
# litellm globals being fully populated during import time.
|
||||
|
|
@ -1830,9 +1841,10 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
|
|||
if params is not None:
|
||||
# Filter out params that are only used for cache key, not for HTTPHandler.__init__
|
||||
handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"}
|
||||
handler_params["client"] = session
|
||||
_new_client = HTTPHandler(**handler_params)
|
||||
else:
|
||||
_new_client = HTTPHandler(timeout=_default_cached_client_timeout())
|
||||
_new_client = HTTPHandler(timeout=_default_cached_client_timeout(), client=session)
|
||||
|
||||
cache.set_cache(
|
||||
key=_cache_key_name,
|
||||
|
|
|
|||
|
|
@ -2172,3 +2172,122 @@ async def test_async_ollama_ssl_verify_false_builds_a_client_that_skips_certific
|
|||
assert route.call_count == 1
|
||||
assert text == "Sunny"
|
||||
assert _cached_clients_verify_modes(client_cache, AsyncHTTPHandler) == (ssl.CERT_NONE,)
|
||||
|
||||
|
||||
def _recording_transport(seen: list[str]) -> httpx.MockTransport:
|
||||
def record(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(str(request.url))
|
||||
return httpx.Response(200, json={})
|
||||
|
||||
return httpx.MockTransport(record)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_async_httpx_client_sends_through_litellm_aclient_session(monkeypatch):
|
||||
"""A configured ``litellm.aclient_session`` is the client the handler sends with, and
|
||||
closing the handler leaves the caller's session open."""
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
seen: list[str] = []
|
||||
session = httpx.AsyncClient(transport=_recording_transport(seen))
|
||||
monkeypatch.setattr(litellm, "aclient_session", session)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
handler = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
|
||||
assert handler.client is session
|
||||
await handler.post("https://example.invalid/v1/messages", json={})
|
||||
assert seen == ["https://example.invalid/v1/messages"]
|
||||
|
||||
await handler.close()
|
||||
assert session.is_closed is False
|
||||
await session.aclose()
|
||||
|
||||
|
||||
def test_get_httpx_client_sends_through_litellm_client_session(monkeypatch):
|
||||
"""Same for the sync side: ``litellm.client_session`` is what ``HTTPHandler`` sends with."""
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client
|
||||
|
||||
seen: list[str] = []
|
||||
session = httpx.Client(transport=_recording_transport(seen))
|
||||
monkeypatch.setattr(litellm, "client_session", session)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
handler = get_httpx_client()
|
||||
assert handler.client is session
|
||||
handler.post("https://example.invalid/v1/messages", json={})
|
||||
assert seen == ["https://example.invalid/v1/messages"]
|
||||
|
||||
handler.close()
|
||||
assert session.is_closed is False
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swapping_litellm_aclient_session_gets_a_fresh_handler(monkeypatch):
|
||||
"""The session is part of the cache key, so a session set after the first call is picked up."""
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
monkeypatch.setattr(litellm, "aclient_session", None)
|
||||
own = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
|
||||
|
||||
session = httpx.AsyncClient(transport=_recording_transport([]))
|
||||
monkeypatch.setattr(litellm, "aclient_session", session)
|
||||
through_session = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
|
||||
|
||||
assert own.client is not session
|
||||
assert through_session.client is session
|
||||
await own.close()
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_explicit_transport_wins_over_litellm_aclient_session(monkeypatch):
|
||||
"""A caller that passes its own transport, like the in-process ASGI client, keeps it
|
||||
when a session is configured, so its requests never leave through the session."""
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
session_seen: list[str] = []
|
||||
transport_seen: list[str] = []
|
||||
session = httpx.AsyncClient(transport=_recording_transport(session_seen))
|
||||
monkeypatch.setattr(litellm, "aclient_session", session)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
handler = get_async_httpx_client(
|
||||
llm_provider=LlmProviders.ANTHROPIC, params={"transport": _recording_transport(transport_seen)}
|
||||
)
|
||||
await handler.post("https://example.invalid/v1/messages", json={})
|
||||
|
||||
assert handler.client is not session
|
||||
assert transport_seen == ["https://example.invalid/v1/messages"]
|
||||
assert session_seen == []
|
||||
await handler.close()
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_anthropic_goes_through_litellm_aclient_session(monkeypatch):
|
||||
"""Providers on the HTTP handler path (here Anthropic) send through the configured session,
|
||||
not through a connection of their own: the mock answers, so nothing reaches 127.0.0.1:9."""
|
||||
seen: list[str] = []
|
||||
|
||||
def record(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(str(request.url))
|
||||
return httpx.Response(400, json={"error": {"type": "invalid_request_error", "message": "mock"}})
|
||||
|
||||
session = httpx.AsyncClient(transport=httpx.MockTransport(record))
|
||||
monkeypatch.setattr(litellm, "aclient_session", session)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
await litellm.acompletion(
|
||||
model="anthropic/claude-3-5-haiku-20241022",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="sk-test",
|
||||
api_base="http://127.0.0.1:9",
|
||||
)
|
||||
assert seen == ["http://127.0.0.1:9/v1/messages"]
|
||||
await session.aclose()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue