diff --git a/litellm/caching/evicted_client_closer.py b/litellm/caching/evicted_client_closer.py index b22cd3aecd1..239a6725523 100644 --- a/litellm/caching/evicted_client_closer.py +++ b/litellm/caching/evicted_client_closer.py @@ -139,6 +139,9 @@ def _has_connection_in_flight(client: object) -> bool: if getattr(getattr(client, "connection_pool", None), "_in_use_connections", None): return True transport: Final = _transport_of(client) + spread_in_flight: Final[object] = getattr(transport, "has_request_in_flight", None) + if callable(spread_in_flight): + return bool(spread_in_flight()) pooled_busy: Final = _pool_has_busy_connection(transport) if pooled_busy is not None: return pooled_busy diff --git a/litellm/llms/custom_httpx/http2_connection_spread.py b/litellm/llms/custom_httpx/http2_connection_spread.py index 0fc1d8c38b9..b52e47ebc6a 100644 --- a/litellm/llms/custom_httpx/http2_connection_spread.py +++ b/litellm/llms/custom_httpx/http2_connection_spread.py @@ -74,6 +74,21 @@ class _AsyncReleasingStream(httpx.AsyncByteStream): await self._stream.aclose() +def _allowed_over_http2(name: bytes, value: bytes) -> bool: + return name.lower() != b"te" or value.strip().lower() == b"trailers" + + +def _without_te_codings(request: httpx.Request) -> httpx.Request: + if request.url.scheme != "https": + return request + kept: Final = tuple((name, value) for name, value in request.headers.raw if _allowed_over_http2(name, value)) + if len(kept) == len(request.headers.raw): + return request + return httpx.Request( + request.method, request.url, headers=kept, stream=request.stream, extensions=request.extensions + ) + + def _with_stream(response: httpx.Response, stream: httpx.AsyncByteStream) -> httpx.Response: return httpx.Response(response.status_code, headers=response.headers, stream=stream, extensions=response.extensions) @@ -90,7 +105,7 @@ class AsyncHTTP2ConnectionSpread(httpx.AsyncBaseTransport): async def handle_async_request(self, request: httpx.Request) -> httpx.Response: lane: Final = self._lanes.acquire() try: - response: Final = await lane.transport.handle_async_request(request) + response: Final = await lane.transport.handle_async_request(_without_te_codings(request)) except BaseException: lane.release() raise @@ -100,6 +115,9 @@ class AsyncHTTP2ConnectionSpread(httpx.AsyncBaseTransport): return response return _with_stream(response, _AsyncReleasingStream(stream, lane.release)) + def has_request_in_flight(self) -> bool: + return any(lane.in_flight > 0 for lane in self._lanes.lanes) + async def aclose(self) -> None: for lane in self._lanes.lanes: await lane.transport.aclose() diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index e3ba3a086b1..495a5778473 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -86,10 +86,15 @@ def http2_enabled() -> bool: return litellm.http2 is True or str_to_bool(os.getenv("LITELLM_HTTP2", "False")) is True -def http2_disabled() -> bool: - from litellm.secret_managers.main import str_to_bool +_HTTP2_OFF_VALUES: Final = frozenset({"false", "0", "no", "off"}) - return litellm.http2 is False or str_to_bool(os.getenv("LITELLM_HTTP2")) is False + +def _is_http2_off(value: bool | str | None) -> bool: + return value is False or (isinstance(value, str) and value.strip().lower() in _HTTP2_OFF_VALUES) + + +def http2_disabled() -> bool: + return _is_http2_off(litellm.http2) or _is_http2_off(os.getenv("LITELLM_HTTP2")) def _resolve_http2(provider_default: bool, ssl_verify: VerifyTypes | None) -> bool: @@ -467,6 +472,11 @@ def get_ssl_configuration( return ssl_verify +def _httpx_default_verify_unless_configured(verify: VerifyTypes) -> VerifyTypes: + litellm_defaults: Final = (get_ssl_configuration(True), get_ssl_configuration(True, http2=True)) + return True if any(verify is default for default in litellm_defaults) else verify + + _shared_realtime_ssl_context: bool | str | ssl.SSLContext | None = None @@ -1438,15 +1448,20 @@ class AsyncHTTPHandler: - [Default] Otherwise it will return None """ local_address: Final = _IPV4_LOCAL_ADDRESS if litellm.force_ipv4 else None + transport_verify: Final = ( + _httpx_default_verify_unless_configured(verify) + if litellm.force_ipv4 and not AsyncHTTPHandler.should_use_aiohttp_transport() + else verify + ) if http2 or http2_enabled(): return AsyncHTTP2ConnectionSpread( lambda lane_limits: AsyncHTTPTransport( - local_address=local_address, verify=verify, cert=cert, http2=True, limits=lane_limits + local_address=local_address, verify=transport_verify, cert=cert, http2=True, limits=lane_limits ), limits, ) if litellm.force_ipv4: - return AsyncHTTPTransport(local_address=local_address, verify=verify, cert=cert, limits=limits) + return AsyncHTTPTransport(local_address=local_address, verify=transport_verify, cert=cert, limits=limits) return None @staticmethod @@ -1790,7 +1805,12 @@ class HTTPHandler: Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them """ if litellm.force_ipv4: - return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, verify=verify, cert=cert, http2=http2) + return HTTPTransport( + local_address=_IPV4_LOCAL_ADDRESS, + verify=_httpx_default_verify_unless_configured(verify), + cert=cert, + http2=http2, + ) return getattr(litellm, "sync_transport", None) @staticmethod diff --git a/tests/integration/sdk/test_http2_wire.py b/tests/integration/sdk/test_http2_wire.py index 8ea94e67739..c224afac3c3 100644 --- a/tests/integration/sdk/test_http2_wire.py +++ b/tests/integration/sdk/test_http2_wire.py @@ -9,6 +9,7 @@ import json import queue import socket import ssl +import subprocess import threading import time from collections.abc import Callable, Generator, Iterator @@ -40,6 +41,7 @@ from integration._support.vertex import service_account_json STREAM_CHUNKS: Final = 3 HELD_SEARCHES: Final = 150 HOLD_SECONDS: Final = 10.0 +PROBED_HEADERS: Final = frozenset({b"te", b"keep-alive", b"connection", b"proxy-connection", b"upgrade", b"x-probe"}) VERTEX_SERVING_CONFIG_PATH: Final = ( "/v1/projects/p/locations/global/collections/default_collection/engines/e/servingConfigs/default_search" ) @@ -48,6 +50,7 @@ VERTEX_SERVING_CONFIG_PATH: Final = ( @dataclass(frozen=True, slots=True) class PeerLog: held_arrivals: queue.SimpleQueue[int] = field(default_factory=queue.SimpleQueue) + all_held_arrived: asyncio.Event = field(default_factory=asyncio.Event) token_requests: queue.SimpleQueue[str] = field(default_factory=queue.SimpleQueue) counted_client_ports: queue.SimpleQueue[int] = field(default_factory=queue.SimpleQueue) @@ -122,6 +125,10 @@ async def _peer(log: PeerLog, scope: Scope, receive: ASGIReceiveCallable, send: log.counted_client_ports.put(counted_client[1] if counted_client else 0) await _send_json(send, {"results": [{"id": "counted"}]}) return + if scope["path"].startswith("/headers/") and scope["path"].endswith(":search"): + seen: Final = sorted(name.decode() for name, _ in scope["headers"] if name in PROBED_HEADERS) + await _send_json(send, {"results": [{"id": f"http/{version}"}, *({"id": name} for name in seen)]}) + return if scope["path"].endswith(":search"): await _send_json(send, {"results": [{"id": f"http/{version}"}]}) return @@ -144,10 +151,12 @@ async def _peer(log: PeerLog, scope: Scope, receive: ASGIReceiveCallable, send: async def _hold_until_all_arrive(log: PeerLog, client_port: int) -> str: log.held_arrivals.put(client_port) - deadline: Final = time.monotonic() + HOLD_SECONDS - while log.held_arrivals.qsize() < HELD_SEARCHES and time.monotonic() < deadline: - await asyncio.sleep(0.01) - return "held-together" if log.held_arrivals.qsize() >= HELD_SEARCHES else "released-alone" + if log.held_arrivals.qsize() >= HELD_SEARCHES: + log.all_held_arrived.set() + waiter: Final = asyncio.ensure_future(log.all_held_arrived.wait()) + await asyncio.wait((waiter,), timeout=HOLD_SECONDS) + waiter.cancel() + return "held-together" if log.all_held_arrived.is_set() else "released-alone" async def _send_json(send: ASGISendCallable, body: dict[str, object]) -> None: @@ -191,11 +200,11 @@ def http2_tls_peer(tmp_path_factory: pytest.TempPathFactory) -> Iterator[TlsPeer thread.join(timeout=10) -def _async_exchange(base_url: str) -> Observed: +def _async_exchange(base_url: str, ssl_verify: bool | None = False) -> Observed: from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler async def _run() -> Observed: - handler: Final = AsyncHTTPHandler(ssl_verify=False) + handler: Final = AsyncHTTPHandler(ssl_verify=ssl_verify) try: response: Final = await handler.client.post(f"{base_url}/echo", json={"ping": "pong"}) async with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response: @@ -211,10 +220,10 @@ def _async_exchange(base_url: str) -> Observed: return asyncio.run(_run()) -def _sync_exchange(base_url: str) -> Observed: +def _sync_exchange(base_url: str, ssl_verify: bool | None = False) -> Observed: from litellm.llms.custom_httpx.http_handler import HTTPHandler - handler: Final = HTTPHandler(ssl_verify=False) + handler: Final = HTTPHandler(ssl_verify=ssl_verify) try: response: Final = handler.client.post(f"{base_url}/echo", json={"ping": "pong"}) with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response: @@ -292,7 +301,10 @@ def _vertex_search_file_ids( async def _avertex_search_file_ids( - peer: TlsPeer, search_url: str | None = None, ssl_verify: ssl.SSLContext | None = None + peer: TlsPeer, + search_url: str | None = None, + ssl_verify: ssl.SSLContext | None = None, + extra_headers: dict[str, object] | None = None, ) -> list[str | None]: import litellm @@ -304,6 +316,7 @@ async def _avertex_search_file_ids( vertex_credentials=service_account_json("p", peer.url), vertex_project="p", ssl_verify=ssl_verify, + extra_headers=extra_headers, ) return [result.get("file_id") for result in response.get("data") or ()] @@ -383,21 +396,48 @@ def test_vertex_search_over_forced_ipv4_trusts_the_configured_ca( assert asyncio.run(_avertex_search_file_ids(http2_tls_peer)) == ["http/2"] +@pytest.mark.parametrize("off", ["False", "0", "no", " OFF "]) def test_explicit_http2_off_keeps_vertex_search_on_http1( - monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer + monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, off: str +) -> None: + _trust_peer_without_global_http2(monkeypatch, http2_tls_peer) + monkeypatch.setenv("LITELLM_HTTP2", off) + assert asyncio.run(_avertex_search_file_ids(http2_tls_peer)) == ["http/1.1"] + + +@pytest.mark.parametrize("off", [False, "false"]) +def test_http2_setting_off_keeps_vertex_search_on_http1( + monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, off: bool | str ) -> None: import litellm _trust_peer_without_global_http2(monkeypatch, http2_tls_peer) - monkeypatch.setenv("LITELLM_HTTP2", "False") - assert _vertex_search_file_ids(http2_tls_peer) == ["http/1.1"] + monkeypatch.setattr(litellm, "http2", off) assert asyncio.run(_avertex_search_file_ids(http2_tls_peer)) == ["http/1.1"] - monkeypatch.delenv("LITELLM_HTTP2") - monkeypatch.setattr(litellm, "http2", False) - litellm.in_memory_llm_clients_cache.flush_cache() - assert _vertex_search_file_ids(http2_tls_peer) == ["http/1.1"] - assert asyncio.run(_avertex_search_file_ids(http2_tls_peer)) == ["http/1.1"] + +@pytest.mark.parametrize( + ("te", "forwarded"), [("gzip", ["http/2", "x-probe"]), ("trailers", ["http/2", "te", "x-probe"])] +) +def test_async_vertex_search_over_http2_sends_no_connection_specific_headers( + monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, te: str, forwarded: list[str] +) -> None: + _trust_peer_without_global_http2(monkeypatch, http2_tls_peer) + received: Final = asyncio.run( + _avertex_search_file_ids( + http2_tls_peer, + search_url=f"{http2_tls_peer.url}/headers", + extra_headers={ + "TE": te, + "Keep-Alive": "timeout=5", + "Connection": "keep-alive", + "Proxy-Connection": "keep-alive", + "Upgrade": "h2c", + "X-Probe": "kept", + }, + ) + ) + assert received == forwarded def _alpn_offered_by(context: ssl.SSLContext, peer: TlsPeer) -> str | None: @@ -665,6 +705,30 @@ def test_forced_ipv4_handlers_honour_ssl_verify_false(monkeypatch: pytest.Monkey _assert_negotiated(_sync_exchange(http2_tls_peer.url), enabled) +def _ca_dir_holding(cert_file: Path, ca_dir: Path) -> Path: + subject_hash: Final = subprocess.run( + ["openssl", "x509", "-hash", "-noout", "-in", str(cert_file)], capture_output=True, text=True, check=True + ).stdout.strip() + (ca_dir / f"{subject_hash}.0").write_bytes(cert_file.read_bytes()) + return ca_dir + + +def test_forced_ipv4_handlers_trust_a_ca_found_only_in_ssl_cert_dir( + monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, tmp_path: Path +) -> None: + import litellm + + monkeypatch.setattr(litellm, "force_ipv4", True) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.delenv("SSL_CERT_FILE", raising=False) + monkeypatch.setenv("SSL_CERT_DIR", str(_ca_dir_holding(http2_tls_peer.cert_file, tmp_path))) + for enabled in (False, True): + _set_http2(monkeypatch, enabled) + _assert_negotiated(_async_exchange(http2_tls_peer.url, ssl_verify=None), enabled) + _assert_negotiated(_sync_exchange(http2_tls_peer.url, ssl_verify=None), enabled) + + def test_vertex_search_over_forced_ipv4_presents_the_client_certificate( monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, tmp_path: Path ) -> None: diff --git a/tests/unit/caching/test_evicted_client_closer.py b/tests/unit/caching/test_evicted_client_closer.py index 7679e276621..1a4619a14c9 100644 --- a/tests/unit/caching/test_evicted_client_closer.py +++ b/tests/unit/caching/test_evicted_client_closer.py @@ -356,13 +356,14 @@ async def test_a_client_serving_a_request_is_not_closed_when_its_grace_window_en @pytest.mark.asyncio -async def test_the_aiohttp_backed_handler_is_not_closed_mid_request(): - """The default async path is aiohttp-backed, whose pool accounts for its own leases.""" +@pytest.mark.parametrize("http2", [False, True], ids=["aiohttp", "http2-connection-spread"]) +async def test_a_handler_is_not_closed_mid_request(http2: bool): + """Both async handler paths account for their own leases: aiohttp's pool and the HTTP/2 connection spread.""" server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0) port = server.sockets[0].getsockname()[1] clock = FakeClock() closer = make_closer(clock) - handler = AsyncHTTPHandler() + handler = AsyncHTTPHandler(http2=http2) held_client = handler.client closer.mark_owned(handler)