Merge pull request #41268 from BerriAI/litellm_outbound_http2_opt_in

feat(http): opt-in outbound HTTP/2 for httpx clients

litellm/__init__.py

@@ -525,6 +525,7 @@ aiohttp_trust_env: bool = False  # set to true to use HTTP_ Proxy settings
 disable_aiohttp_transport: bool = False  # Set this to true to use httpx instead
 disable_aiohttp_trust_env: bool = False  # When False, aiohttp will respect HTTP(S)_PROXY env vars
 force_ipv4: bool = False  # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
+http2: bool = False
 network_mock: bool = False  # When True, use mock transport — no real network calls
 
 ####### STOP SEQUENCE LIMIT #######

litellm/llms/custom_httpx/http_handler.py

@@ -74,6 +74,12 @@ _IPV4_LOCAL_ADDRESS: Final = "0.0.0.0"
 _HttpxTransportT = TypeVar("_HttpxTransportT", HTTPTransport, AsyncHTTPTransport)
 
 
+def http2_enabled() -> bool:
+    from litellm.secret_managers.main import str_to_bool
+
+    return litellm.http2 is True or str_to_bool(os.getenv("LITELLM_HTTP2", "False")) is True
+
+
 def _environment_proxy_mounts(
     build_proxy_transport: Callable[[str], _HttpxTransportT],
 ) -> Mapping[str, _HttpxTransportT | None]:
@@ -638,6 +644,7 @@ class AsyncHTTPHandler:
             headers=default_headers,
             cookies=blocked_cookie_jar(),
             follow_redirects=True,
+            http2=http2_enabled(),
         )
 
     async def close(self):
@@ -1157,6 +1164,10 @@ class AsyncHTTPHandler:
 
         from litellm.secret_managers.main import str_to_bool
 
+        if http2_enabled():
+            verbose_logger.debug("LITELLM_HTTP2 enabled, using httpx transport (aiohttp has no HTTP/2 support)")
+            return False
+
         #########################################################
         # Check if user disabled aiohttp transport
         ########################################################
@@ -1287,7 +1298,7 @@ class AsyncHTTPHandler:
         - [Default] If force_ipv4 is False, it will return None
         """
         if litellm.force_ipv4:
-            return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS)
+            return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled())
         else:
             return None
 
@@ -1300,7 +1311,7 @@ class AsyncHTTPHandler:
         if not isinstance(transport, AsyncHTTPTransport):
             return None
         return _environment_proxy_mounts(
-            lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert)
+            lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled())
         )
 
 
@@ -1342,6 +1353,7 @@ class HTTPHandler:
             headers=default_headers,
             cookies=blocked_cookie_jar(),
             follow_redirects=True,
+            http2=http2_enabled(),
         )
 
     @property
@@ -1616,7 +1628,7 @@ class HTTPHandler:
         Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
         """
         if litellm.force_ipv4:
-            return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS)
+            return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled())
         else:
             return getattr(litellm, "sync_transport", None)
 
@@ -1627,7 +1639,9 @@ class HTTPHandler:
     ) -> Mapping[str, HTTPTransport | None] | None:
         if not litellm.force_ipv4:
             return None
-        return _environment_proxy_mounts(lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert))
+        return _environment_proxy_mounts(
+            lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled())
+        )
 
 
 def get_async_httpx_client(

litellm/llms/openai/common_utils.py

@@ -32,6 +32,7 @@ from litellm.llms.custom_httpx.http_handler import (
     _DEFAULT_TTL_FOR_HTTPX_CLIENTS,
     AsyncHTTPHandler,
     get_ssl_configuration,
+    http2_enabled,
 )
 
 
@@ -325,6 +326,7 @@ class BaseOpenAILLM:
             transport=transport,
             mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=None),
             follow_redirects=True,
+            http2=http2_enabled(),
         )
 
     @staticmethod
@@ -343,6 +345,7 @@ class BaseOpenAILLM:
         return httpx.Client(
             verify=ssl_config,
             follow_redirects=True,
+            http2=http2_enabled(),
         )
 
 

litellm/main.py

@@ -105,7 +105,7 @@ from litellm.llms.base_llm.base_model_iterator import (
 )
 from litellm.llms.bedrock.common_utils import BedrockModelInfo
 from litellm.llms.cohere.common_utils import CohereModelInfo
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
 from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
 from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 from litellm.llms.vertex_ai.common_utils import (
@@ -2341,6 +2341,10 @@ def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
 def _complete_aiohttp_openai(
     ctx: _CompletionDispatchContext,
 ) -> _CompletionDispatchResult:
+    if http2_enabled():
+        verbose_logger.warning(
+            "litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1"
+        )
     acompletion: Final = ctx.acompletion
     api_base = ctx.api_base
     api_key = ctx.api_key

pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     # When changing a floor, verify it installs + imports on every supported
     # Python with: `uv pip install --resolution=lowest-direct .`
     "fastuuid>=0.14.0,<1.0",
-    "httpx>=0.28.0,<1.0",
+    "httpx[http2]>=0.28.0,<1.0",
     "openai>=2.20.0,<3.0.0",
     "python-dotenv>=1.0.0,<2.0",
     "tiktoken>=0.8.0,<1.0",

tests/e2e/llm_translation/test_outbound_http2_e2e.py

@@ -0,0 +1,208 @@
+"""Outbound HTTP/2 negotiation for LiteLLM-built httpx clients.
+
+Spins up a local hypercorn TLS server that offers h2 and http/1.1 over ALPN and
+drives the real AsyncHTTPHandler / HTTPHandler at it, so the negotiated protocol
+on the wire is the assertion. No running proxy or provider credentials needed,
+which is why these tests carry no `e2e` marker (same shape as the markerless
+harness checks under tests/e2e/load/).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+import ipaddress
+import socket
+import threading
+import time
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Final, cast
+
+import pytest
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+from hypercorn.asyncio import (
+    serve,  # pyright: ignore[reportUnknownVariableType]  # hypercorn's serve signature passes through untyped worker hooks
+)
+from hypercorn.config import Config
+from hypercorn.typing import (
+    ASGIReceiveCallable,
+    ASGISendCallable,
+    HTTPResponseBodyEvent,
+    HTTPResponseStartEvent,
+    Scope,
+)
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+
+
+def _write_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]:
+    key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    now: Final = datetime.datetime.now(datetime.timezone.utc)
+    name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])
+    cert: Final = (
+        x509.CertificateBuilder()
+        .subject_name(name)
+        .issuer_name(name)
+        .public_key(key.public_key())
+        .serial_number(x509.random_serial_number())
+        .not_valid_before(now - datetime.timedelta(days=1))
+        .not_valid_after(now + datetime.timedelta(days=7))
+        .add_extension(
+            x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]),
+            critical=False,
+        )
+        .sign(key, hashes.SHA256())
+    )
+    cert_file: Final = cert_dir / "cert.pem"
+    key_file: Final = cert_dir / "key.pem"
+    cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
+    key_file.write_bytes(
+        key.private_bytes(
+            serialization.Encoding.PEM,
+            serialization.PrivateFormat.TraditionalOpenSSL,
+            serialization.NoEncryption(),
+        )
+    )
+    return cert_file, key_file
+
+
+async def _asgi_app(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None:
+    if scope["type"] != "http":
+        return
+    while True:
+        message = await receive()
+        if message["type"] == "http.disconnect":
+            return
+        if message["type"] == "http.request" and not message["more_body"]:
+            break
+    if scope["path"] == "/stream":
+        await send(
+            HTTPResponseStartEvent(
+                type="http.response.start", status=200, headers=[(b"content-type", b"text/event-stream")]
+            )
+        )
+        for index in range(3):
+            await send(
+                HTTPResponseBodyEvent(
+                    type="http.response.body", body=f"data: chunk-{index}\n\n".encode(), more_body=True
+                )
+            )
+        await send(HTTPResponseBodyEvent(type="http.response.body", body=b"", more_body=False))
+        return
+    await send(
+        HTTPResponseStartEvent(type="http.response.start", status=200, headers=[(b"content-type", b"application/json")])
+    )
+    await send(HTTPResponseBodyEvent(type="http.response.body", body=b'{"ok": true}', more_body=False))
+
+
+@pytest.fixture(scope="module")
+def http2_tls_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]:
+    cert_dir: Final = tmp_path_factory.mktemp("h2certs")
+    cert_file, key_file = _write_self_signed_cert(cert_dir)
+
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        port: Final = cast(int, sock.getsockname()[1])
+
+    shutdown: Final = threading.Event()
+
+    def _serve() -> None:
+        loop: Final = asyncio.new_event_loop()
+        config: Final = Config()
+        config.bind = [f"127.0.0.1:{port}"]
+        config.certfile = str(cert_file)
+        config.keyfile = str(key_file)
+        config.alpn_protocols = ["h2", "http/1.1"]
+        loop.run_until_complete(serve(_asgi_app, config, shutdown_trigger=lambda: asyncio.to_thread(shutdown.wait)))
+        loop.close()
+
+    thread: Final = threading.Thread(target=_serve, daemon=True)
+    thread.start()
+
+    for _ in range(100):
+        try:
+            with socket.create_connection(("127.0.0.1", port), timeout=0.2):
+                break
+        except OSError:
+            time.sleep(0.05)
+    else:
+        pytest.fail("hypercorn test server did not start")
+
+    yield f"https://127.0.0.1:{port}"
+
+    shutdown.set()
+    thread.join(timeout=10)
+
+
+def _async_exchange(base_url: str) -> tuple[str, str, bytes]:
+    async def _run() -> tuple[str, str, bytes]:
+        handler: Final = AsyncHTTPHandler(ssl_verify=False)
+        try:
+            response: Final = await handler.client.post(f"{base_url}/echo", json={"ping": "pong"})
+            post_version: Final = response.http_version
+            async with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response:
+                stream_version: Final = stream_response.http_version
+                body: Final = b"".join([chunk async for chunk in stream_response.aiter_bytes()])
+            return post_version, stream_version, body
+        finally:
+            await handler.close()
+
+    return asyncio.run(_run())
+
+
+def _sync_exchange(base_url: str) -> tuple[str, str, bytes]:
+    handler: Final = HTTPHandler(ssl_verify=False)
+    try:
+        response: Final = handler.client.post(f"{base_url}/echo", json={"ping": "pong"})
+        post_version: Final = response.http_version
+        with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response:
+            stream_version: Final = stream_response.http_version
+            body: Final = b"".join(stream_response.iter_bytes())
+        return post_version, stream_version, body
+    finally:
+        handler.close()
+
+
+class TestOutboundHttp2:
+    @pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")])
+    def test_async_handler_negotiates_http2_only_when_enabled(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        http2_tls_server: str,
+        use_http2: bool,
+        expected_version: str,
+    ) -> None:
+        monkeypatch.setattr(litellm, "http2", use_http2)
+        monkeypatch.delenv("LITELLM_HTTP2", raising=False)
+        monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+        monkeypatch.setattr(litellm, "force_ipv4", False)
+
+        post_version, stream_version, body = _async_exchange(http2_tls_server)
+
+        assert post_version == expected_version
+        assert stream_version == expected_version
+        assert b"data: chunk-0" in body
+
+    @pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")])
+    def test_sync_handler_negotiates_http2_only_when_enabled(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        http2_tls_server: str,
+        use_http2: bool,
+        expected_version: str,
+    ) -> None:
+        monkeypatch.setattr(litellm, "http2", use_http2)
+        monkeypatch.delenv("LITELLM_HTTP2", raising=False)
+        monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+        monkeypatch.setattr(litellm, "force_ipv4", False)
+
+        post_version, stream_version, body = _sync_exchange(http2_tls_server)
+
+        assert post_version == expected_version
+        assert stream_version == expected_version
+        assert b"data: chunk-0" in body

tests/test_litellm/llms/custom_httpx/test_http_handler.py

@@ -1675,3 +1675,30 @@ async def test_bounded_get_closes_stream_on_cancellation(respx_mock, monkeypatch
     finally:
         await handler.close()
     assert closed.is_set()
+
+
+@pytest.mark.asyncio
+async def test_http2_flag_bypasses_aiohttp_transport(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", False)
+    monkeypatch.setattr(litellm, "force_ipv4", False)
+    monkeypatch.delenv("LITELLM_HTTP2", raising=False)
+    monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False)
+
+    monkeypatch.setattr(litellm, "http2", True)
+    assert AsyncHTTPHandler._should_use_aiohttp_transport() is False
+    assert AsyncHTTPHandler._create_async_transport() is None
+
+    monkeypatch.setattr(litellm, "http2", False)
+    monkeypatch.setenv("LITELLM_HTTP2", "True")
+    assert AsyncHTTPHandler._should_use_aiohttp_transport() is False
+    assert AsyncHTTPHandler._create_async_transport() is None
+
+
+@pytest.mark.asyncio
+async def test_http2_disabled_by_default(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(litellm, "http2", False)
+    monkeypatch.delenv("LITELLM_HTTP2", raising=False)
+    monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False)
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", False)
+
+    assert AsyncHTTPHandler._should_use_aiohttp_transport() is True

tests/test_litellm/llms/openai/test_openai_common_utils.py

@@ -411,3 +411,5 @@ async def test_async_genuine_bad_request_still_raises(provider, stream):
 )
 def test_is_openai_backed_api_base_decides_by_hostname_only(api_base, expected):
     assert is_openai_backed_api_base(api_base) is expected
+
+

tests/test_litellm/test_main.py

@@ -4,6 +4,7 @@ from datetime import datetime
 import contextlib
 import copy
 import json
+import logging
 import os
 from collections.abc import Mapping
 from dataclasses import dataclass
@@ -3850,3 +3851,27 @@ def test_bridged_responses_with_openai_http_handler_keeps_forwarded_headers_out_
     assert "extra_headers" not in body
     assert body["model"] == "gpt-5.4"
     assert {k: request.headers[k] for k in FORWARDED_CLIENT_HEADERS} == FORWARDED_CLIENT_HEADERS
+
+
+@pytest.mark.parametrize("http2_on", [True, False])
+def test_aiohttp_openai_warns_only_when_http2_enabled(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, http2_on: bool
+):
+    from litellm.main import base_llm_aiohttp_handler
+
+    monkeypatch.setattr(litellm, "http2", http2_on)
+    monkeypatch.delenv("LITELLM_HTTP2", raising=False)
+
+    handler_completion: Final = MagicMock(return_value=MagicMock())
+    monkeypatch.setattr(base_llm_aiohttp_handler, "completion", handler_completion)
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM"):
+        litellm.completion(
+            model="aiohttp_openai/gpt-4o",
+            messages=[{"role": "user", "content": "hi"}],
+            api_key="sk-test",
+        )
+
+    assert handler_completion.called
+    warned: Final = "aiohttp_openai/ always uses aiohttp" in caplog.text
+    assert warned is http2_on

uv.lock

@@ -3295,6 +3295,11 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
 ]
 
+[package.optional-dependencies]
+http2 = [
+    { name = "h2" },
+]
+
 [[package]]
 name = "httpx-sse"
 version = "0.4.3"
@@ -4464,7 +4469,7 @@ dependencies = [
     { name = "boto3" },
     { name = "click" },
     { name = "fastuuid" },
-    { name = "httpx" },
+    { name = "httpx", extra = ["http2"] },
     { name = "importlib-metadata" },
     { name = "jinja2" },
     { name = "jsonschema" },
@@ -4717,7 +4722,7 @@ requires-dist = [
     { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" },
     { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" },
     { name = "hiredis", marker = "extra == 'proxy'", specifier = ">=3.0.0,<4.0" },
-    { name = "httpx", specifier = ">=0.28.0,<1.0" },
+    { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" },
     { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" },
     { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" },
     { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" },