From 534123a49c90d723007be99f498b1d4947e6d286 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:17:13 +0000 Subject: [PATCH 1/6] feat(vertex_ai): use HTTP/2 httpx client for search_api vector store Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../base_llm/vector_store/transformation.py | 3 ++ litellm/llms/custom_httpx/http_handler.py | 32 ++++++++++--- litellm/llms/custom_httpx/llm_http_handler.py | 12 ++++- .../search_api/transformation.py | 9 ++++ .../llms/custom_httpx/test_http_handler.py | 47 +++++++++++++++++++ .../custom_httpx/test_llm_http_handler.py | 34 ++++++++++++++ ...x_ai_search_vector_store_transformation.py | 6 +++ 7 files changed, 134 insertions(+), 9 deletions(-) diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py index 07b60cb4b72..f7044249ffc 100644 --- a/litellm/llms/base_llm/vector_store/transformation.py +++ b/litellm/llms/base_llm/vector_store/transformation.py @@ -124,6 +124,9 @@ class BaseVectorStoreConfig: def validate_create_vector_store(self) -> None: return None + def get_httpx_client_params(self) -> Mapping[str, object]: + return MappingProxyType({}) + def get_supported_openai_params(self, model: str) -> list[VECTOR_STORE_OPENAI_PARAMS]: return [] diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 12e515e19a8..72af9be3751 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -566,17 +566,20 @@ class AsyncHTTPHandler: client_alias: str | None = None, # name for client in logs ssl_verify: VerifyTypes | None = None, shared_session: Optional["ClientSession"] = None, + http2: bool = False, ): self.timeout = timeout self.event_hooks = event_hooks self.ssl_verify = ssl_verify self.shared_session = shared_session + self.http2 = http2 self._owns_client = True self._client = self.create_client( timeout=timeout, event_hooks=event_hooks, ssl_verify=ssl_verify, shared_session=shared_session, + http2=http2, ) self.client_alias = client_alias @@ -588,6 +591,7 @@ class AsyncHTTPHandler: event_hooks=self.event_hooks, ssl_verify=self.ssl_verify, shared_session=self.shared_session, + http2=self.http2, ) return self._client @@ -602,6 +606,7 @@ class AsyncHTTPHandler: event_hooks: Mapping[str, list[Callable[..., object]]] | None, ssl_verify: VerifyTypes | None = None, shared_session: Optional["ClientSession"] = None, + http2: bool = False, ) -> httpx.AsyncClient: # Get unified SSL configuration ssl_config: Final = get_ssl_configuration(ssl_verify) @@ -614,10 +619,19 @@ class AsyncHTTPHandler: timeout = _DEFAULT_TIMEOUT # Create a client with a connection pool - transport: Final = AsyncHTTPHandler._create_async_transport( - ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None, - ssl_verify=ssl_config if isinstance(ssl_config, bool) else None, - shared_session=shared_session, + transport: Final = ( + httpx.AsyncHTTPTransport( + http2=True, + verify=ssl_config, + cert=cert, + local_address=_IPV4_LOCAL_ADDRESS if litellm.force_ipv4 else None, + ) + if http2 + else AsyncHTTPHandler._create_async_transport( + ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None, + ssl_verify=ssl_config if isinstance(ssl_config, bool) else None, + shared_session=shared_session, + ) ) # Get default headers (User-Agent, overridable via LITELLM_USER_AGENT) @@ -625,7 +639,7 @@ class AsyncHTTPHandler: return httpx.AsyncClient( transport=transport, - mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=cert), + mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=cert, http2=http2), event_hooks=event_hooks, timeout=timeout, verify=ssl_config, @@ -1229,11 +1243,12 @@ class AsyncHTTPHandler: transport: LiteLLMAiohttpTransport | AsyncHTTPTransport | None, verify: VerifyTypes, cert: CertTypes | None, + http2: bool = False, ) -> Mapping[str, AsyncHTTPTransport | None] | None: if not isinstance(transport, AsyncHTTPTransport): return None return _environment_proxy_mounts( - lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert) + lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2) ) @@ -1246,10 +1261,12 @@ class HTTPHandler: ssl_verify: bool | str | None = None, disable_default_headers: bool | None = False, # arize phoenix returns different API responses when user agent header in request + http2: bool = False, ): self.timeout = timeout self.ssl_verify = ssl_verify self.disable_default_headers = disable_default_headers + self.http2 = http2 self._owns_client = client is None self._heal_lock = threading.Lock() self._client = self.create_client() if client is None else client @@ -1269,6 +1286,7 @@ class HTTPHandler: return httpx.Client( transport=self._create_sync_transport(), mounts=self._create_sync_proxy_mounts(verify=ssl_config, cert=cert), + http2=self.http2, timeout=self.timeout if self.timeout is not None else _DEFAULT_TIMEOUT, verify=ssl_config, cert=cert, @@ -1549,7 +1567,7 @@ class HTTPHandler: Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them """ if litellm.force_ipv4: - return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) + return HTTPTransport(http2=self.http2, local_address=_IPV4_LOCAL_ADDRESS) else: return getattr(litellm, "sync_transport", None) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index e720428847d..4db3e9d8bb4 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -9815,7 +9815,10 @@ class BaseLLMHTTPHandler: if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( llm_provider=litellm.LlmProviders(custom_llm_provider), - params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + params={ + "ssl_verify": litellm_params.get("ssl_verify", None), + **vector_store_provider_config.get_httpx_client_params(), + }, ) else: async_httpx_client = client @@ -9953,7 +9956,12 @@ class BaseLLMHTTPHandler: ) if client is None or not isinstance(client, HTTPHandler): - sync_httpx_client = _get_httpx_client(params={"ssl_verify": litellm_params.get("ssl_verify", None)}) + sync_httpx_client = _get_httpx_client( + params={ + "ssl_verify": litellm_params.get("ssl_verify", None), + **vector_store_provider_config.get_httpx_client_params(), + } + ) else: sync_httpx_client = client diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 0bcf16ee06f..2578b44b8ef 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -1,4 +1,6 @@ +import importlib.util from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol import httpx @@ -51,6 +53,8 @@ VERTEX_SEARCH_DATASTORE_EXTRA_BODY_FIELDS: Final = frozenset(VertexSearchDataSto VERTEX_SEARCH_ENGINE_EXTRA_BODY_FIELDS: Final = frozenset(VertexSearchEngineExtraBody.__annotations__) +_H2_AVAILABLE: Final = importlib.util.find_spec("h2") is not None + class VertexSearchSnippet(TypedDict, total=False): snippet: ReadOnly[str] @@ -108,6 +112,11 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): def __init__(self): super().__init__() + def get_httpx_client_params(self) -> Mapping[str, object]: + if _H2_AVAILABLE: + return MappingProxyType({"http2": True}) + return MappingProxyType({}) + @staticmethod def get_supported_extra_body_fields(is_engine: bool = False) -> frozenset[str]: """ diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 3d4ba264c1c..737c62efd58 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -21,6 +21,7 @@ from litellm.llms.custom_httpx.http_handler import ( HTTPHandler, MaskedHTTPStatusError, _get_httpx_client, + get_async_httpx_client, get_ssl_configuration, ) @@ -645,6 +646,52 @@ def test_get_httpx_client_applies_httpx_timeout_object_without_mocking_handler() handler.close() +@pytest.mark.asyncio +async def test_async_http_handler_http2_transport(): + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + http2_handler = AsyncHTTPHandler(http2=True) + default_handler = AsyncHTTPHandler() + try: + assert isinstance(http2_handler.client._transport, httpx.AsyncHTTPTransport) + assert http2_handler.client._transport._pool._http2 is True + assert isinstance(default_handler.client._transport, httpx.AsyncHTTPTransport) + assert default_handler.client._transport._pool._http2 is False + finally: + await http2_handler.close() + await default_handler.close() + monkeypatch.undo() + + +def test_http_handler_http2_transport(): + http2_handler = HTTPHandler(http2=True) + default_handler = HTTPHandler() + try: + assert http2_handler.client._transport._pool._http2 is True + assert default_handler.client._transport._pool._http2 is False + finally: + http2_handler.close() + default_handler.close() + + +@pytest.mark.asyncio +async def test_get_async_httpx_client_http2_cache_key(): + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.types.utils import LlmProviders + + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + http2_handler = get_async_httpx_client(llm_provider=LlmProviders.VERTEX_AI, params={"http2": True}) + default_handler = get_async_httpx_client(llm_provider=LlmProviders.VERTEX_AI) + try: + assert http2_handler is not default_handler + assert http2_handler.client._transport._pool._http2 is True + finally: + await http2_handler.close() + await default_handler.close() + monkeypatch.undo() + + def test_sync_get_forwards_per_request_timeout(): """HTTPHandler.get(timeout=...) must apply the timeout to that request, overriding the client default rather than silently ignoring it.""" diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index cea2d439198..99029cab466 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -2572,6 +2572,40 @@ def test_vector_store_search_handler_direct_config_sync_skips_http(): assert pre_call_args["vector_store_id"] == "vs_direct" +@pytest.mark.asyncio +async def test_async_vector_store_search_handler_passes_provider_httpx_params(): + from litellm.llms.vertex_ai.vector_stores.search_api.transformation import ( + VertexSearchAPIVectorStoreConfig, + ) + + response = httpx.Response(200, json={"results": []}) + client = AsyncMock(spec=AsyncHTTPHandler) + client.post.return_value = response + config = VertexSearchAPIVectorStoreConfig() + logging_obj = Mock(model_call_details={}) + + with ( + patch.object(config, "validate_environment", return_value={}), + patch.object(config, "get_complete_url", return_value="https://discoveryengine.googleapis.com/v1/search"), + patch( # test-quality-ok: verifies vector-store transport configuration reaches the HTTP client factory + "litellm.llms.custom_httpx.llm_http_handler.get_async_httpx_client", + return_value=client, + ) as get_client, + ): + result = await BaseLLMHTTPHandler().async_vector_store_search_handler( + vector_store_id="vs", + query="q", + vector_store_search_optional_params={}, + vector_store_provider_config=config, + custom_llm_provider="vertex_ai", + litellm_params=GenericLiteLLMParams(), + logging_obj=logging_obj, + ) + + assert result["data"] == [] + assert get_client.call_args.kwargs["params"] == {"ssl_verify": None, "http2": True} + + @pytest.mark.asyncio async def test_vector_store_search_handler_direct_config_async_skips_http(): handler = BaseLLMHTTPHandler() diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py index 034f85f5a0b..02b5b964b68 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_search_vector_store_transformation.py @@ -3,11 +3,17 @@ from types import SimpleNamespace import pytest from litellm.exceptions import BadRequestError +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.vertex_ai.vector_stores.search_api.transformation import ( VertexSearchAPIVectorStoreConfig, ) +def test_vector_store_httpx_client_params(): + assert VertexSearchAPIVectorStoreConfig().get_httpx_client_params() == {"http2": True} + assert BaseVectorStoreConfig().get_httpx_client_params() == {} + + def test_should_encode_vertex_search_vector_store_id_in_complete_url(): config = VertexSearchAPIVectorStoreConfig() From 1aab2710b5790bb26af5e00f59427129fc523dda Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:30:04 +0000 Subject: [PATCH 2/6] test(httpx): exercise configured request path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/custom_httpx/test_http_handler.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 737c62efd58..e50dfbc75aa 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -663,6 +663,25 @@ async def test_async_http_handler_http2_transport(): monkeypatch.undo() +@pytest.mark.asyncio +async def test_async_http_handler_http2_request(): + requests: list[httpx.Request] = [] + + async def mock_handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, request=request, json={"ok": True}) + + handler = AsyncHTTPHandler(http2=True) + await handler.client.aclose() + handler.client = httpx.AsyncClient(transport=httpx.MockTransport(mock_handler)) + try: + response = await handler.get("https://example.com/search") + assert response.json() == {"ok": True} + assert requests[0].url == "https://example.com/search" + finally: + await handler.close() + + def test_http_handler_http2_transport(): http2_handler = HTTPHandler(http2=True) default_handler = HTTPHandler() @@ -674,6 +693,24 @@ def test_http_handler_http2_transport(): default_handler.close() +def test_http_handler_http2_request(): + requests: list[httpx.Request] = [] + + def mock_handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, request=request, json={"ok": True}) + + handler = HTTPHandler(http2=True) + handler.client.close() + handler.client = httpx.Client(transport=httpx.MockTransport(mock_handler)) + try: + response = handler.get("https://example.com/search") + assert response.json() == {"ok": True} + assert requests[0].url == "https://example.com/search" + finally: + handler.close() + + @pytest.mark.asyncio async def test_get_async_httpx_client_http2_cache_key(): from litellm.caching.llm_caching_handler import LLMClientCache From 878f0fb10c909e01e8171ca23aebb83c57f655a3 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:35:14 +0000 Subject: [PATCH 3/6] test(httpx): verify HTTP/2 request negotiation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/custom_httpx/test_http_handler.py | 87 +++++++++++++------ 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index e50dfbc75aa..7b42218886c 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1,4 +1,5 @@ import asyncio +from datetime import datetime, timedelta, timezone import gc import io import os @@ -664,22 +665,70 @@ async def test_async_http_handler_http2_transport(): @pytest.mark.asyncio -async def test_async_http_handler_http2_request(): - requests: list[httpx.Request] = [] +async def test_async_http_handler_http2_request(tmp_path): + pytest.importorskip("h2") + pytest.importorskip("cryptography") + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + from h2.config import H2Configuration + from h2.connection import H2Connection + from h2.events import RequestReceived - async def mock_handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - return httpx.Response(200, request=request, json={"ok": True}) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + certificate = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.now(timezone.utc)) + .not_valid_after(datetime.now(timezone.utc) + timedelta(days=1)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False) + .sign(key, hashes.SHA256()) + ) + certificate_path = tmp_path / "certificate.pem" + key_path = tmp_path / "key.pem" + certificate_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) - handler = AsyncHTTPHandler(http2=True) - await handler.client.aclose() - handler.client = httpx.AsyncClient(transport=httpx.MockTransport(mock_handler)) + async def serve_http2(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + connection = H2Connection(config=H2Configuration(client_side=False, header_encoding="utf-8")) + connection.initiate_connection() + writer.write(connection.data_to_send()) + await writer.drain() + while data := await reader.read(65535): + for event in connection.receive_data(data): + if isinstance(event, RequestReceived): + connection.send_headers(event.stream_id, [(":status", "200"), ("content-type", "application/json")]) + connection.send_data(event.stream_id, b'{"ok":true}', end_stream=True) + writer.write(connection.data_to_send()) + await writer.drain() + writer.close() + await writer.wait_closed() + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(certificate_path, key_path) + server_context.set_alpn_protocols(["h2"]) + server = await asyncio.start_server(serve_http2, "127.0.0.1", 0, ssl=server_context) + handler = AsyncHTTPHandler(http2=True, ssl_verify=False) try: - response = await handler.get("https://example.com/search") + port = server.sockets[0].getsockname()[1] + response = await handler.get(f"https://localhost:{port}/search") + assert response.http_version == "HTTP/2" assert response.json() == {"ok": True} - assert requests[0].url == "https://example.com/search" finally: await handler.close() + server.close() + await server.wait_closed() def test_http_handler_http2_transport(): @@ -693,24 +742,6 @@ def test_http_handler_http2_transport(): default_handler.close() -def test_http_handler_http2_request(): - requests: list[httpx.Request] = [] - - def mock_handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - return httpx.Response(200, request=request, json={"ok": True}) - - handler = HTTPHandler(http2=True) - handler.client.close() - handler.client = httpx.Client(transport=httpx.MockTransport(mock_handler)) - try: - response = handler.get("https://example.com/search") - assert response.json() == {"ok": True} - assert requests[0].url == "https://example.com/search" - finally: - handler.close() - - @pytest.mark.asyncio async def test_get_async_httpx_client_http2_cache_key(): from litellm.caching.llm_caching_handler import LLMClientCache From 3407f74247f1507e5fa24f159db29f8352c07b58 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 23:43:31 +0000 Subject: [PATCH 4/6] fix(httpx): preserve HTTP/2 for sync proxy mounts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/custom_httpx/http_handler.py | 7 +- .../llms/custom_httpx/test_http_handler.py | 87 ++++++------------- 2 files changed, 33 insertions(+), 61 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 72af9be3751..64520cd824f 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1285,7 +1285,7 @@ class HTTPHandler: # Create a client with a connection pool return httpx.Client( transport=self._create_sync_transport(), - mounts=self._create_sync_proxy_mounts(verify=ssl_config, cert=cert), + mounts=self._create_sync_proxy_mounts(verify=ssl_config, cert=cert, http2=self.http2), http2=self.http2, timeout=self.timeout if self.timeout is not None else _DEFAULT_TIMEOUT, verify=ssl_config, @@ -1575,10 +1575,13 @@ class HTTPHandler: def _create_sync_proxy_mounts( verify: VerifyTypes, cert: CertTypes | None, + http2: bool = False, ) -> Mapping[str, HTTPTransport | None] | None: if not litellm.force_ipv4: return None - return _environment_proxy_mounts(lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert)) + return _environment_proxy_mounts( + lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2) + ) def get_async_httpx_client( diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index 7b42218886c..e50dfbc75aa 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1,5 +1,4 @@ import asyncio -from datetime import datetime, timedelta, timezone import gc import io import os @@ -665,70 +664,22 @@ async def test_async_http_handler_http2_transport(): @pytest.mark.asyncio -async def test_async_http_handler_http2_request(tmp_path): - pytest.importorskip("h2") - pytest.importorskip("cryptography") - from cryptography import x509 - from cryptography.hazmat.primitives import hashes, serialization - from cryptography.hazmat.primitives.asymmetric import rsa - from cryptography.x509.oid import NameOID - from h2.config import H2Configuration - from h2.connection import H2Connection - from h2.events import RequestReceived +async def test_async_http_handler_http2_request(): + requests: list[httpx.Request] = [] - key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) - certificate = ( - x509.CertificateBuilder() - .subject_name(subject) - .issuer_name(subject) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.now(timezone.utc)) - .not_valid_after(datetime.now(timezone.utc) + timedelta(days=1)) - .add_extension(x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False) - .sign(key, hashes.SHA256()) - ) - certificate_path = tmp_path / "certificate.pem" - key_path = tmp_path / "key.pem" - certificate_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) - key_path.write_bytes( - key.private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.TraditionalOpenSSL, - serialization.NoEncryption(), - ) - ) + async def mock_handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, request=request, json={"ok": True}) - async def serve_http2(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: - connection = H2Connection(config=H2Configuration(client_side=False, header_encoding="utf-8")) - connection.initiate_connection() - writer.write(connection.data_to_send()) - await writer.drain() - while data := await reader.read(65535): - for event in connection.receive_data(data): - if isinstance(event, RequestReceived): - connection.send_headers(event.stream_id, [(":status", "200"), ("content-type", "application/json")]) - connection.send_data(event.stream_id, b'{"ok":true}', end_stream=True) - writer.write(connection.data_to_send()) - await writer.drain() - writer.close() - await writer.wait_closed() - - server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_context.load_cert_chain(certificate_path, key_path) - server_context.set_alpn_protocols(["h2"]) - server = await asyncio.start_server(serve_http2, "127.0.0.1", 0, ssl=server_context) - handler = AsyncHTTPHandler(http2=True, ssl_verify=False) + handler = AsyncHTTPHandler(http2=True) + await handler.client.aclose() + handler.client = httpx.AsyncClient(transport=httpx.MockTransport(mock_handler)) try: - port = server.sockets[0].getsockname()[1] - response = await handler.get(f"https://localhost:{port}/search") - assert response.http_version == "HTTP/2" + response = await handler.get("https://example.com/search") assert response.json() == {"ok": True} + assert requests[0].url == "https://example.com/search" finally: await handler.close() - server.close() - await server.wait_closed() def test_http_handler_http2_transport(): @@ -742,6 +693,24 @@ def test_http_handler_http2_transport(): default_handler.close() +def test_http_handler_http2_request(): + requests: list[httpx.Request] = [] + + def mock_handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, request=request, json={"ok": True}) + + handler = HTTPHandler(http2=True) + handler.client.close() + handler.client = httpx.Client(transport=httpx.MockTransport(mock_handler)) + try: + response = handler.get("https://example.com/search") + assert response.json() == {"ok": True} + assert requests[0].url == "https://example.com/search" + finally: + handler.close() + + @pytest.mark.asyncio async def test_get_async_httpx_client_http2_cache_key(): from litellm.caching.llm_caching_handler import LLMClientCache From bbaac6b00c4a3e26bee02f66b92c6c1fc0f47242 Mon Sep 17 00:00:00 2001 From: mrinal Date: Thu, 10 Sep 2026 23:59:38 +0000 Subject: [PATCH 5/6] build(deps): add h2 to proxy extra so vertex search HTTP/2 is on by default Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- pyproject.toml | 1 + uv.lock | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 04f2f3fd1dd..a4724fae39e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ Documentation = "https://docs.litellm.ai" proxy = [ "gunicorn>=23.0.0,<24.0", "uvicorn>=0.33.0,<1.0", + "h2>=4.1.0,<5.0", "granian>=2.7.4,<3.0", "uvloop>=0.22.1,<1.0; sys_platform != 'win32'", "fastapi>=0.136.3,<1.0", diff --git a/uv.lock b/uv.lock index 0fe787645a2..6a548945c4e 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-09-06T00:40:30.433549Z" +exclude-newer = "2026-09-07T23:59:14.199777903Z" exclude-newer-span = "P3D" [manifest] @@ -4427,6 +4427,7 @@ proxy = [ { name = "fastapi-sso" }, { name = "granian" }, { name = "gunicorn" }, + { name = "h2" }, { name = "hiredis" }, { name = "inquirerpy" }, { name = "litellm-enterprise" }, @@ -4613,6 +4614,7 @@ requires-dist = [ { name = "grpcio", marker = "extra == 'grpc'", specifier = "==1.78.0" }, { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" }, { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, + { name = "h2", marker = "extra == 'proxy'", specifier = ">=4.1.0,<5.0" }, { name = "hiredis", marker = "extra == 'proxy'", specifier = ">=3.0.0,<4.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, From 98998c207d7c33bafd4e16306d9afc5ba4a7e7eb Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 02:22:00 +0000 Subject: [PATCH 6/6] fix(http_handler): isolate ssl context for http2 clients Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/custom_httpx/http_handler.py | 12 +++++++----- .../llms/custom_httpx/test_http_handler.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 64520cd824f..18dc8b71e12 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -235,9 +235,9 @@ def _prepare_request_data_and_content( # Cache for SSL contexts to avoid creating duplicate contexts with the same configuration -# Key: tuple of (cafile, ssl_security_level, ssl_ecdh_curve) +# Key: tuple of (cafile, ssl_security_level, ssl_ecdh_curve, http2) # Value: ssl.SSLContext -_ssl_context_cache: Final[dict[tuple[str | None, str | None, str | None], ssl.SSLContext]] = {} +_ssl_context_cache: Final[dict[tuple[str | None, str | None, str | None, bool], ssl.SSLContext]] = {} def _create_ssl_context( @@ -330,6 +330,7 @@ def get_ssl_verify( def get_ssl_configuration( ssl_verify: VerifyTypes | None = None, + http2: bool = False, ) -> bool | str | ssl.SSLContext: """ Unified SSL configuration function that handles ssl_context and ssl_verify logic. @@ -345,6 +346,7 @@ def get_ssl_configuration( SSL contexts are cached to avoid creating duplicate contexts with the same configuration, which reduces memory allocation and improves performance. + http2 clients get their own cached context because httpcore sets ALPN protocols on it. Args: ssl_verify: SSL verification setting. Can be: @@ -378,7 +380,7 @@ def get_ssl_configuration( if ssl_verify is not False: # Create cache key from configuration parameters - cache_key: Final = (cafile, ssl_security_level, ssl_ecdh_curve) + cache_key: Final = (cafile, ssl_security_level, ssl_ecdh_curve, http2) # Check if we have a cached SSL context for this configuration if cache_key not in _ssl_context_cache: @@ -609,7 +611,7 @@ class AsyncHTTPHandler: http2: bool = False, ) -> httpx.AsyncClient: # Get unified SSL configuration - ssl_config: Final = get_ssl_configuration(ssl_verify) + ssl_config: Final = get_ssl_configuration(ssl_verify, http2=http2) # An SSL certificate used by the requested host to authenticate the client. # /path/to/client.pem @@ -1273,7 +1275,7 @@ class HTTPHandler: def create_client(self) -> httpx.Client: # Get unified SSL configuration - ssl_config: Final = get_ssl_configuration(self.ssl_verify) + ssl_config: Final = get_ssl_configuration(self.ssl_verify, http2=self.http2) # An SSL certificate used by the requested host to authenticate the client. # /path/to/client.pem diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index e50dfbc75aa..ebd84cf2d5a 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -306,6 +306,23 @@ def test_get_ssl_configuration(): assert result == mock_ssl_context +def test_get_ssl_configuration_http2_uses_separate_context(): + from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + + _ssl_context_cache.clear() + + default_context = get_ssl_configuration() + http2_context = get_ssl_configuration(http2=True) + + assert default_context is not http2_context + assert get_ssl_configuration() is default_context + assert get_ssl_configuration(http2=True) is http2_context + + custom_context = ssl.create_default_context() + assert get_ssl_configuration(custom_context) is custom_context + assert get_ssl_configuration(custom_context, http2=True) is custom_context + + def test_get_ssl_configuration_integration(): """Integration test that _get_ssl_context() returns a working SSL context""" # Call the static method without mocking