diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index cffeb44f12c..eea07aa1534 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -633,6 +633,7 @@ class AsyncHTTPHandler: shared_session: Optional["ClientSession"] = None, transport: httpx.AsyncBaseTransport | None = None, follow_redirects: bool = True, + client: httpx.AsyncClient | None = None, ): self.timeout = timeout self.event_hooks = event_hooks @@ -640,12 +641,16 @@ class AsyncHTTPHandler: self.shared_session = shared_session self.transport = transport self.follow_redirects = follow_redirects - self._owns_client = True - self._client = self.create_client( - timeout=timeout, - event_hooks=event_hooks, - ssl_verify=ssl_verify, - shared_session=shared_session, + self._owns_client = client is None + self._client = ( + client + if client is not None + else self.create_client( + timeout=timeout, + event_hooks=event_hooks, + ssl_verify=ssl_verify, + shared_session=shared_session, + ) ) self.client_alias = client_alias @@ -1762,7 +1767,10 @@ def get_async_httpx_client( except Exception: pass - _cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider + # An explicit transport (in-process ASGI, pinned egress) wins over the session; the key holds the session so a swap never serves a stale handler. + session: Final = litellm.aclient_session if params is None or "transport" not in params else None + _session_key_name: Final = f"_session{id(session)}" if session is not None else "" + _cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider + _session_key_name # Lazily initialize the global in-memory client cache to avoid relying on # litellm globals being fully populated during import time. @@ -1781,11 +1789,12 @@ def get_async_httpx_client( # Filter out params that are only used for cache key, not for AsyncHTTPHandler.__init__ handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"} handler_params["shared_session"] = shared_session - _new_client = AsyncHTTPHandler(**handler_params) + _new_client = AsyncHTTPHandler(**handler_params, client=session) else: _new_client = AsyncHTTPHandler( timeout=_default_cached_client_timeout(), shared_session=shared_session, + client=session, ) cache.set_cache( @@ -1812,7 +1821,9 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler: except Exception: pass - _cache_key_name: Final = "httpx_client" + _params_key_name + session: Final = litellm.client_session + _session_key_name: Final = f"_session{id(session)}" if session is not None else "" + _cache_key_name: Final = "httpx_client" + _params_key_name + _session_key_name # Lazily initialize the global in-memory client cache to avoid relying on # litellm globals being fully populated during import time. @@ -1830,9 +1841,10 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler: if params is not None: # Filter out params that are only used for cache key, not for HTTPHandler.__init__ handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"} + handler_params["client"] = session _new_client = HTTPHandler(**handler_params) else: - _new_client = HTTPHandler(timeout=_default_cached_client_timeout()) + _new_client = HTTPHandler(timeout=_default_cached_client_timeout(), client=session) cache.set_cache( key=_cache_key_name, diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index bd4ed4c99cc..dc7bf563514 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -2172,3 +2172,122 @@ async def test_async_ollama_ssl_verify_false_builds_a_client_that_skips_certific assert route.call_count == 1 assert text == "Sunny" assert _cached_clients_verify_modes(client_cache, AsyncHTTPHandler) == (ssl.CERT_NONE,) + + +def _recording_transport(seen: list[str]) -> httpx.MockTransport: + def record(request: httpx.Request) -> httpx.Response: + seen.append(str(request.url)) + return httpx.Response(200, json={}) + + return httpx.MockTransport(record) + + +@pytest.mark.asyncio +async def test_get_async_httpx_client_sends_through_litellm_aclient_session(monkeypatch): + """A configured ``litellm.aclient_session`` is the client the handler sends with, and + closing the handler leaves the caller's session open.""" + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + seen: list[str] = [] + session = httpx.AsyncClient(transport=_recording_transport(seen)) + monkeypatch.setattr(litellm, "aclient_session", session) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) + assert handler.client is session + await handler.post("https://example.invalid/v1/messages", json={}) + assert seen == ["https://example.invalid/v1/messages"] + + await handler.close() + assert session.is_closed is False + await session.aclose() + + +def test_get_httpx_client_sends_through_litellm_client_session(monkeypatch): + """Same for the sync side: ``litellm.client_session`` is what ``HTTPHandler`` sends with.""" + from litellm.llms.custom_httpx.http_handler import get_httpx_client + + seen: list[str] = [] + session = httpx.Client(transport=_recording_transport(seen)) + monkeypatch.setattr(litellm, "client_session", session) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler = get_httpx_client() + assert handler.client is session + handler.post("https://example.invalid/v1/messages", json={}) + assert seen == ["https://example.invalid/v1/messages"] + + handler.close() + assert session.is_closed is False + session.close() + + +@pytest.mark.asyncio +async def test_swapping_litellm_aclient_session_gets_a_fresh_handler(monkeypatch): + """The session is part of the cache key, so a session set after the first call is picked up.""" + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(litellm, "aclient_session", None) + own = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) + + session = httpx.AsyncClient(transport=_recording_transport([])) + monkeypatch.setattr(litellm, "aclient_session", session) + through_session = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) + + assert own.client is not session + assert through_session.client is session + await own.close() + await session.aclose() + + +@pytest.mark.asyncio +async def test_an_explicit_transport_wins_over_litellm_aclient_session(monkeypatch): + """A caller that passes its own transport, like the in-process ASGI client, keeps it + when a session is configured, so its requests never leave through the session.""" + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + session_seen: list[str] = [] + transport_seen: list[str] = [] + session = httpx.AsyncClient(transport=_recording_transport(session_seen)) + monkeypatch.setattr(litellm, "aclient_session", session) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler = get_async_httpx_client( + llm_provider=LlmProviders.ANTHROPIC, params={"transport": _recording_transport(transport_seen)} + ) + await handler.post("https://example.invalid/v1/messages", json={}) + + assert handler.client is not session + assert transport_seen == ["https://example.invalid/v1/messages"] + assert session_seen == [] + await handler.close() + await session.aclose() + + +@pytest.mark.asyncio +async def test_acompletion_anthropic_goes_through_litellm_aclient_session(monkeypatch): + """Providers on the HTTP handler path (here Anthropic) send through the configured session, + not through a connection of their own: the mock answers, so nothing reaches 127.0.0.1:9.""" + seen: list[str] = [] + + def record(request: httpx.Request) -> httpx.Response: + seen.append(str(request.url)) + return httpx.Response(400, json={"error": {"type": "invalid_request_error", "message": "mock"}}) + + session = httpx.AsyncClient(transport=httpx.MockTransport(record)) + monkeypatch.setattr(litellm, "aclient_session", session) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + with pytest.raises(litellm.BadRequestError): + await litellm.acompletion( + model="anthropic/claude-3-5-haiku-20241022", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-test", + api_base="http://127.0.0.1:9", + ) + assert seen == ["http://127.0.0.1:9/v1/messages"] + await session.aclose()