This commit is contained in:
Max Azatian 2026-10-25 13:26:39 +00:00 • committed by GitHub
commit 5106df6e07
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 141 additions and 10 deletions

View file

@ -633,6 +633,7 @@ class AsyncHTTPHandler:
shared_session: Optional["ClientSession"] = None,
transport: httpx.AsyncBaseTransport | None = None,
follow_redirects: bool = True,
client: httpx.AsyncClient | None = None,
):
self.timeout = timeout
self.event_hooks = event_hooks
@ -640,12 +641,16 @@ class AsyncHTTPHandler:
self.shared_session = shared_session
self.transport = transport
self.follow_redirects = follow_redirects
self._owns_client = True
self._client = self.create_client(
timeout=timeout,
event_hooks=event_hooks,
ssl_verify=ssl_verify,
shared_session=shared_session,
self._owns_client = client is None
self._client = (
client
if client is not None
else self.create_client(
timeout=timeout,
event_hooks=event_hooks,
ssl_verify=ssl_verify,
shared_session=shared_session,
)
)
self.client_alias = client_alias
@ -1762,7 +1767,10 @@ def get_async_httpx_client(
except Exception:
pass
_cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider
# An explicit transport (in-process ASGI, pinned egress) wins over the session; the key holds the session so a swap never serves a stale handler.
session: Final = litellm.aclient_session if params is None or "transport" not in params else None
_session_key_name: Final = f"_session{id(session)}" if session is not None else ""
_cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider + _session_key_name
# Lazily initialize the global in-memory client cache to avoid relying on
# litellm globals being fully populated during import time.
@ -1781,11 +1789,12 @@ def get_async_httpx_client(
# Filter out params that are only used for cache key, not for AsyncHTTPHandler.__init__
handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"}
handler_params["shared_session"] = shared_session
_new_client = AsyncHTTPHandler(**handler_params)
_new_client = AsyncHTTPHandler(**handler_params, client=session)
else:
_new_client = AsyncHTTPHandler(
timeout=_default_cached_client_timeout(),
shared_session=shared_session,
client=session,
)
cache.set_cache(
@ -1812,7 +1821,9 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
except Exception:
pass
_cache_key_name: Final = "httpx_client" + _params_key_name
session: Final = litellm.client_session
_session_key_name: Final = f"_session{id(session)}" if session is not None else ""
_cache_key_name: Final = "httpx_client" + _params_key_name + _session_key_name
# Lazily initialize the global in-memory client cache to avoid relying on
# litellm globals being fully populated during import time.
@ -1830,9 +1841,10 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
if params is not None:
# Filter out params that are only used for cache key, not for HTTPHandler.__init__
handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"}
handler_params["client"] = session
_new_client = HTTPHandler(**handler_params)
else:
_new_client = HTTPHandler(timeout=_default_cached_client_timeout())
_new_client = HTTPHandler(timeout=_default_cached_client_timeout(), client=session)
cache.set_cache(
key=_cache_key_name,

View file

@ -2172,3 +2172,122 @@ async def test_async_ollama_ssl_verify_false_builds_a_client_that_skips_certific
assert route.call_count == 1
assert text == "Sunny"
assert _cached_clients_verify_modes(client_cache, AsyncHTTPHandler) == (ssl.CERT_NONE,)
def _recording_transport(seen: list[str]) -> httpx.MockTransport:
def record(request: httpx.Request) -> httpx.Response:
seen.append(str(request.url))
return httpx.Response(200, json={})
return httpx.MockTransport(record)
@pytest.mark.asyncio
async def test_get_async_httpx_client_sends_through_litellm_aclient_session(monkeypatch):
"""A configured ``litellm.aclient_session`` is the client the handler sends with, and
closing the handler leaves the caller's session open."""
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.utils import LlmProviders
seen: list[str] = []
session = httpx.AsyncClient(transport=_recording_transport(seen))
monkeypatch.setattr(litellm, "aclient_session", session)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
handler = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
assert handler.client is session
await handler.post("https://example.invalid/v1/messages", json={})
assert seen == ["https://example.invalid/v1/messages"]
await handler.close()
assert session.is_closed is False
await session.aclose()
def test_get_httpx_client_sends_through_litellm_client_session(monkeypatch):
"""Same for the sync side: ``litellm.client_session`` is what ``HTTPHandler`` sends with."""
from litellm.llms.custom_httpx.http_handler import get_httpx_client
seen: list[str] = []
session = httpx.Client(transport=_recording_transport(seen))
monkeypatch.setattr(litellm, "client_session", session)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
handler = get_httpx_client()
assert handler.client is session
handler.post("https://example.invalid/v1/messages", json={})
assert seen == ["https://example.invalid/v1/messages"]
handler.close()
assert session.is_closed is False
session.close()
@pytest.mark.asyncio
async def test_swapping_litellm_aclient_session_gets_a_fresh_handler(monkeypatch):
"""The session is part of the cache key, so a session set after the first call is picked up."""
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.utils import LlmProviders
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
monkeypatch.setattr(litellm, "aclient_session", None)
own = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
session = httpx.AsyncClient(transport=_recording_transport([]))
monkeypatch.setattr(litellm, "aclient_session", session)
through_session = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC)
assert own.client is not session
assert through_session.client is session
await own.close()
await session.aclose()
@pytest.mark.asyncio
async def test_an_explicit_transport_wins_over_litellm_aclient_session(monkeypatch):
"""A caller that passes its own transport, like the in-process ASGI client, keeps it
when a session is configured, so its requests never leave through the session."""
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.utils import LlmProviders
session_seen: list[str] = []
transport_seen: list[str] = []
session = httpx.AsyncClient(transport=_recording_transport(session_seen))
monkeypatch.setattr(litellm, "aclient_session", session)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
handler = get_async_httpx_client(
llm_provider=LlmProviders.ANTHROPIC, params={"transport": _recording_transport(transport_seen)}
)
await handler.post("https://example.invalid/v1/messages", json={})
assert handler.client is not session
assert transport_seen == ["https://example.invalid/v1/messages"]
assert session_seen == []
await handler.close()
await session.aclose()
@pytest.mark.asyncio
async def test_acompletion_anthropic_goes_through_litellm_aclient_session(monkeypatch):
"""Providers on the HTTP handler path (here Anthropic) send through the configured session,
not through a connection of their own: the mock answers, so nothing reaches 127.0.0.1:9."""
seen: list[str] = []
def record(request: httpx.Request) -> httpx.Response:
seen.append(str(request.url))
return httpx.Response(400, json={"error": {"type": "invalid_request_error", "message": "mock"}})
session = httpx.AsyncClient(transport=httpx.MockTransport(record))
monkeypatch.setattr(litellm, "aclient_session", session)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
with pytest.raises(litellm.BadRequestError):
await litellm.acompletion(
model="anthropic/claude-3-5-haiku-20241022",
messages=[{"role": "user", "content": "hi"}],
api_key="sk-test",
api_base="http://127.0.0.1:9",
)
assert seen == ["http://127.0.0.1:9/v1/messages"]
await session.aclose()