From 38a690752c8c0db6c8fc71a96f7298b2a683d301 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Sun, 18 Oct 2026 02:10:03 +0200 Subject: [PATCH 1/2] fix(http_handler): send through litellm.aclient_session and client_session on the HTTP handler path litellm.aclient_session and litellm.client_session were read only by the OpenAI and Azure SDK clients. Every provider on the HTTP handler path (Anthropic, OpenRouter, Gemini, ...) built its own httpx client, so a proxy, TLS or egress setting on the session covered OpenAI only. get_async_httpx_client and get_httpx_client now build the handler around the configured session. AsyncHTTPHandler takes a client the way HTTPHandler already does and never closes a caller's client. The session's identity is part of the cache key, so a session set after the first call gets a fresh handler. --- litellm/llms/custom_httpx/http_handler.py | 31 ++++-- .../llms/custom_httpx/test_http_handler.py | 94 +++++++++++++++++++ 2 files changed, 116 insertions(+), 9 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index cffeb44f12c..29b774a7853 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -633,6 +633,7 @@ class AsyncHTTPHandler: shared_session: Optional["ClientSession"] = None, transport: httpx.AsyncBaseTransport | None = None, follow_redirects: bool = True, + client: httpx.AsyncClient | None = None, ): self.timeout = timeout self.event_hooks = event_hooks @@ -640,12 +641,16 @@ class AsyncHTTPHandler: self.shared_session = shared_session self.transport = transport self.follow_redirects = follow_redirects - self._owns_client = True - self._client = self.create_client( - timeout=timeout, - event_hooks=event_hooks, - ssl_verify=ssl_verify, - shared_session=shared_session, + self._owns_client = client is None + self._client = ( + client + if client is not None + else self.create_client( + timeout=timeout, + event_hooks=event_hooks, + ssl_verify=ssl_verify, + shared_session=shared_session, + ) ) self.client_alias = client_alias @@ -1762,7 +1767,10 @@ def get_async_httpx_client( except Exception: pass - _cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider + # A caller-supplied session is part of the key, so swapping it never serves a stale handler. + session: Final = litellm.aclient_session + _session_key_name: Final = f"_session{id(session)}" if session is not None else "" + _cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider + _session_key_name # Lazily initialize the global in-memory client cache to avoid relying on # litellm globals being fully populated during import time. @@ -1781,11 +1789,13 @@ def get_async_httpx_client( # Filter out params that are only used for cache key, not for AsyncHTTPHandler.__init__ handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"} handler_params["shared_session"] = shared_session + handler_params["client"] = session _new_client = AsyncHTTPHandler(**handler_params) else: _new_client = AsyncHTTPHandler( timeout=_default_cached_client_timeout(), shared_session=shared_session, + client=session, ) cache.set_cache( @@ -1812,7 +1822,9 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler: except Exception: pass - _cache_key_name: Final = "httpx_client" + _params_key_name + session: Final = litellm.client_session + _session_key_name: Final = f"_session{id(session)}" if session is not None else "" + _cache_key_name: Final = "httpx_client" + _params_key_name + _session_key_name # Lazily initialize the global in-memory client cache to avoid relying on # litellm globals being fully populated during import time. @@ -1830,9 +1842,10 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler: if params is not None: # Filter out params that are only used for cache key, not for HTTPHandler.__init__ handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"} + handler_params["client"] = session _new_client = HTTPHandler(**handler_params) else: - _new_client = HTTPHandler(timeout=_default_cached_client_timeout()) + _new_client = HTTPHandler(timeout=_default_cached_client_timeout(), client=session) cache.set_cache( key=_cache_key_name, diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index bd4ed4c99cc..5c28cc0630e 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -2172,3 +2172,97 @@ async def test_async_ollama_ssl_verify_false_builds_a_client_that_skips_certific assert route.call_count == 1 assert text == "Sunny" assert _cached_clients_verify_modes(client_cache, AsyncHTTPHandler) == (ssl.CERT_NONE,) + + +def _recording_transport(seen: list[str]) -> httpx.MockTransport: + def record(request: httpx.Request) -> httpx.Response: + seen.append(str(request.url)) + return httpx.Response(200, json={}) + + return httpx.MockTransport(record) + + +@pytest.mark.asyncio +async def test_get_async_httpx_client_sends_through_litellm_aclient_session(monkeypatch): + """A configured ``litellm.aclient_session`` is the client the handler sends with, and + closing the handler leaves the caller's session open.""" + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + seen: list[str] = [] + session = httpx.AsyncClient(transport=_recording_transport(seen)) + monkeypatch.setattr(litellm, "aclient_session", session) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) + assert handler.client is session + await handler.post("https://example.invalid/v1/messages", json={}) + assert seen == ["https://example.invalid/v1/messages"] + + await handler.close() + assert session.is_closed is False + await session.aclose() + + +def test_get_httpx_client_sends_through_litellm_client_session(monkeypatch): + """Same for the sync side: ``litellm.client_session`` is what ``HTTPHandler`` sends with.""" + from litellm.llms.custom_httpx.http_handler import get_httpx_client + + seen: list[str] = [] + session = httpx.Client(transport=_recording_transport(seen)) + monkeypatch.setattr(litellm, "client_session", session) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler = get_httpx_client() + assert handler.client is session + handler.post("https://example.invalid/v1/messages", json={}) + assert seen == ["https://example.invalid/v1/messages"] + + handler.close() + assert session.is_closed is False + session.close() + + +@pytest.mark.asyncio +async def test_swapping_litellm_aclient_session_gets_a_fresh_handler(monkeypatch): + """The session is part of the cache key, so a session set after the first call is picked up.""" + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(litellm, "aclient_session", None) + own = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) + + session = httpx.AsyncClient(transport=_recording_transport([])) + monkeypatch.setattr(litellm, "aclient_session", session) + through_session = get_async_httpx_client(llm_provider=LlmProviders.ANTHROPIC) + + assert own.client is not session + assert through_session.client is session + await own.close() + await session.aclose() + + +@pytest.mark.asyncio +async def test_acompletion_anthropic_goes_through_litellm_aclient_session(monkeypatch): + """Providers on the HTTP handler path (here Anthropic) send through the configured session, + not through a connection of their own: the mock answers, so nothing reaches 127.0.0.1:9.""" + seen: list[str] = [] + + def record(request: httpx.Request) -> httpx.Response: + seen.append(str(request.url)) + return httpx.Response(400, json={"error": {"type": "invalid_request_error", "message": "mock"}}) + + session = httpx.AsyncClient(transport=httpx.MockTransport(record)) + monkeypatch.setattr(litellm, "aclient_session", session) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + with pytest.raises(litellm.BadRequestError): + await litellm.acompletion( + model="anthropic/claude-3-5-haiku-20241022", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-test", + api_base="http://127.0.0.1:9", + ) + assert seen == ["http://127.0.0.1:9/v1/messages"] + await session.aclose() From dce51ce3cb27226809551419f8cbce0a348f0401 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Sun, 25 Oct 2026 14:26:33 +0100 Subject: [PATCH 2/2] fix(http_handler): keep an explicit transport when litellm.aclient_session is set get_async_asgi_client and the ROI calculator pass their own transport (in-process ASGI dispatch, pinned egress). Handing those handlers the global session replaced the transport, so their requests went out over the session. The session now applies only when no transport is given, and client= is passed outside the untyped params dict so the type gate count stays at the merge base. --- litellm/llms/custom_httpx/http_handler.py | 7 +++--- .../llms/custom_httpx/test_http_handler.py | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 29b774a7853..eea07aa1534 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1767,8 +1767,8 @@ def get_async_httpx_client( except Exception: pass - # A caller-supplied session is part of the key, so swapping it never serves a stale handler. - session: Final = litellm.aclient_session + # An explicit transport (in-process ASGI, pinned egress) wins over the session; the key holds the session so a swap never serves a stale handler. + session: Final = litellm.aclient_session if params is None or "transport" not in params else None _session_key_name: Final = f"_session{id(session)}" if session is not None else "" _cache_key_name: Final = "async_httpx_client" + _params_key_name + llm_provider + _session_key_name @@ -1789,8 +1789,7 @@ def get_async_httpx_client( # Filter out params that are only used for cache key, not for AsyncHTTPHandler.__init__ handler_params: Final = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"} handler_params["shared_session"] = shared_session - handler_params["client"] = session - _new_client = AsyncHTTPHandler(**handler_params) + _new_client = AsyncHTTPHandler(**handler_params, client=session) else: _new_client = AsyncHTTPHandler( timeout=_default_cached_client_timeout(), diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index 5c28cc0630e..dc7bf563514 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -2243,6 +2243,31 @@ async def test_swapping_litellm_aclient_session_gets_a_fresh_handler(monkeypatch await session.aclose() +@pytest.mark.asyncio +async def test_an_explicit_transport_wins_over_litellm_aclient_session(monkeypatch): + """A caller that passes its own transport, like the in-process ASGI client, keeps it + when a session is configured, so its requests never leave through the session.""" + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.utils import LlmProviders + + session_seen: list[str] = [] + transport_seen: list[str] = [] + session = httpx.AsyncClient(transport=_recording_transport(session_seen)) + monkeypatch.setattr(litellm, "aclient_session", session) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + + handler = get_async_httpx_client( + llm_provider=LlmProviders.ANTHROPIC, params={"transport": _recording_transport(transport_seen)} + ) + await handler.post("https://example.invalid/v1/messages", json={}) + + assert handler.client is not session + assert transport_seen == ["https://example.invalid/v1/messages"] + assert session_seen == [] + await handler.close() + await session.aclose() + + @pytest.mark.asyncio async def test_acompletion_anthropic_goes_through_litellm_aclient_session(monkeypatch): """Providers on the HTTP handler path (here Anthropic) send through the configured session,