diff --git a/litellm/__init__.py b/litellm/__init__.py index 3668e6efb0c..dde94d68d5f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -525,6 +525,7 @@ aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +http2: bool = False network_mock: bool = False # When True, use mock transport — no real network calls ####### STOP SEQUENCE LIMIT ####### diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index f4883b57fbc..cc37d1d9eca 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -74,6 +74,12 @@ _IPV4_LOCAL_ADDRESS: Final = "0.0.0.0" _HttpxTransportT = TypeVar("_HttpxTransportT", HTTPTransport, AsyncHTTPTransport) +def http2_enabled() -> bool: + from litellm.secret_managers.main import str_to_bool + + return litellm.http2 is True or str_to_bool(os.getenv("LITELLM_HTTP2", "False")) is True + + def _environment_proxy_mounts( build_proxy_transport: Callable[[str], _HttpxTransportT], ) -> Mapping[str, _HttpxTransportT | None]: @@ -638,6 +644,7 @@ class AsyncHTTPHandler: headers=default_headers, cookies=blocked_cookie_jar(), follow_redirects=True, + http2=http2_enabled(), ) async def close(self): @@ -1157,6 +1164,10 @@ class AsyncHTTPHandler: from litellm.secret_managers.main import str_to_bool + if http2_enabled(): + verbose_logger.debug("LITELLM_HTTP2 enabled, using httpx transport (aiohttp has no HTTP/2 support)") + return False + ######################################################### # Check if user disabled aiohttp transport ######################################################## @@ -1287,7 +1298,7 @@ class AsyncHTTPHandler: - [Default] If force_ipv4 is False, it will return None """ if litellm.force_ipv4: - return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) + return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled()) else: return None @@ -1300,7 +1311,7 @@ class AsyncHTTPHandler: if not isinstance(transport, AsyncHTTPTransport): return None return _environment_proxy_mounts( - lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert) + lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled()) ) @@ -1342,6 +1353,7 @@ class HTTPHandler: headers=default_headers, cookies=blocked_cookie_jar(), follow_redirects=True, + http2=http2_enabled(), ) @property @@ -1616,7 +1628,7 @@ class HTTPHandler: Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them """ if litellm.force_ipv4: - return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) + return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled()) else: return getattr(litellm, "sync_transport", None) @@ -1627,7 +1639,9 @@ class HTTPHandler: ) -> Mapping[str, HTTPTransport | None] | None: if not litellm.force_ipv4: return None - return _environment_proxy_mounts(lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert)) + return _environment_proxy_mounts( + lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled()) + ) def get_async_httpx_client( diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 2db6d78a218..cb6a5e4e96a 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -32,6 +32,7 @@ from litellm.llms.custom_httpx.http_handler import ( _DEFAULT_TTL_FOR_HTTPX_CLIENTS, AsyncHTTPHandler, get_ssl_configuration, + http2_enabled, ) @@ -325,6 +326,7 @@ class BaseOpenAILLM: transport=transport, mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=None), follow_redirects=True, + http2=http2_enabled(), ) @staticmethod @@ -343,6 +345,7 @@ class BaseOpenAILLM: return httpx.Client( verify=ssl_config, follow_redirects=True, + http2=http2_enabled(), ) diff --git a/litellm/main.py b/litellm/main.py index 9fbc5881b4f..1c6e47bfb11 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -105,7 +105,7 @@ from litellm.llms.base_llm.base_model_iterator import ( ) from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( @@ -2341,6 +2341,10 @@ def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: def _complete_aiohttp_openai( ctx: _CompletionDispatchContext, ) -> _CompletionDispatchResult: + if http2_enabled(): + verbose_logger.warning( + "litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1" + ) acompletion: Final = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key diff --git a/pyproject.toml b/pyproject.toml index 5f12b3c7307..93ff55c4069 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0", - "httpx>=0.28.0,<1.0", + "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<3.0.0", "python-dotenv>=1.0.0,<2.0", "tiktoken>=0.8.0,<1.0", diff --git a/tests/e2e/llm_translation/test_outbound_http2_e2e.py b/tests/e2e/llm_translation/test_outbound_http2_e2e.py new file mode 100644 index 00000000000..cb2182ffd62 --- /dev/null +++ b/tests/e2e/llm_translation/test_outbound_http2_e2e.py @@ -0,0 +1,208 @@ +"""Outbound HTTP/2 negotiation for LiteLLM-built httpx clients. + +Spins up a local hypercorn TLS server that offers h2 and http/1.1 over ALPN and +drives the real AsyncHTTPHandler / HTTPHandler at it, so the negotiated protocol +on the wire is the assertion. No running proxy or provider credentials needed, +which is why these tests carry no `e2e` marker (same shape as the markerless +harness checks under tests/e2e/load/). +""" + +from __future__ import annotations + +import asyncio +import datetime +import ipaddress +import socket +import threading +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Final, cast + +import pytest +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID +from hypercorn.asyncio import ( + serve, # pyright: ignore[reportUnknownVariableType] # hypercorn's serve signature passes through untyped worker hooks +) +from hypercorn.config import Config +from hypercorn.typing import ( + ASGIReceiveCallable, + ASGISendCallable, + HTTPResponseBodyEvent, + HTTPResponseStartEvent, + Scope, +) + +import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + + +def _write_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now: Final = datetime.datetime.now(datetime.timezone.utc) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=7)) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + cert_file: Final = cert_dir / "cert.pem" + key_file: Final = cert_dir / "key.pem" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return cert_file, key_file + + +async def _asgi_app(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None: + if scope["type"] != "http": + return + while True: + message = await receive() + if message["type"] == "http.disconnect": + return + if message["type"] == "http.request" and not message["more_body"]: + break + if scope["path"] == "/stream": + await send( + HTTPResponseStartEvent( + type="http.response.start", status=200, headers=[(b"content-type", b"text/event-stream")] + ) + ) + for index in range(3): + await send( + HTTPResponseBodyEvent( + type="http.response.body", body=f"data: chunk-{index}\n\n".encode(), more_body=True + ) + ) + await send(HTTPResponseBodyEvent(type="http.response.body", body=b"", more_body=False)) + return + await send( + HTTPResponseStartEvent(type="http.response.start", status=200, headers=[(b"content-type", b"application/json")]) + ) + await send(HTTPResponseBodyEvent(type="http.response.body", body=b'{"ok": true}', more_body=False)) + + +@pytest.fixture(scope="module") +def http2_tls_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: + cert_dir: Final = tmp_path_factory.mktemp("h2certs") + cert_file, key_file = _write_self_signed_cert(cert_dir) + + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port: Final = cast(int, sock.getsockname()[1]) + + shutdown: Final = threading.Event() + + def _serve() -> None: + loop: Final = asyncio.new_event_loop() + config: Final = Config() + config.bind = [f"127.0.0.1:{port}"] + config.certfile = str(cert_file) + config.keyfile = str(key_file) + config.alpn_protocols = ["h2", "http/1.1"] + loop.run_until_complete(serve(_asgi_app, config, shutdown_trigger=lambda: asyncio.to_thread(shutdown.wait))) + loop.close() + + thread: Final = threading.Thread(target=_serve, daemon=True) + thread.start() + + for _ in range(100): + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.05) + else: + pytest.fail("hypercorn test server did not start") + + yield f"https://127.0.0.1:{port}" + + shutdown.set() + thread.join(timeout=10) + + +def _async_exchange(base_url: str) -> tuple[str, str, bytes]: + async def _run() -> tuple[str, str, bytes]: + handler: Final = AsyncHTTPHandler(ssl_verify=False) + try: + response: Final = await handler.client.post(f"{base_url}/echo", json={"ping": "pong"}) + post_version: Final = response.http_version + async with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response: + stream_version: Final = stream_response.http_version + body: Final = b"".join([chunk async for chunk in stream_response.aiter_bytes()]) + return post_version, stream_version, body + finally: + await handler.close() + + return asyncio.run(_run()) + + +def _sync_exchange(base_url: str) -> tuple[str, str, bytes]: + handler: Final = HTTPHandler(ssl_verify=False) + try: + response: Final = handler.client.post(f"{base_url}/echo", json={"ping": "pong"}) + post_version: Final = response.http_version + with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response: + stream_version: Final = stream_response.http_version + body: Final = b"".join(stream_response.iter_bytes()) + return post_version, stream_version, body + finally: + handler.close() + + +class TestOutboundHttp2: + @pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")]) + def test_async_handler_negotiates_http2_only_when_enabled( + self, + monkeypatch: pytest.MonkeyPatch, + http2_tls_server: str, + use_http2: bool, + expected_version: str, + ) -> None: + monkeypatch.setattr(litellm, "http2", use_http2) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", False) + + post_version, stream_version, body = _async_exchange(http2_tls_server) + + assert post_version == expected_version + assert stream_version == expected_version + assert b"data: chunk-0" in body + + @pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")]) + def test_sync_handler_negotiates_http2_only_when_enabled( + self, + monkeypatch: pytest.MonkeyPatch, + http2_tls_server: str, + use_http2: bool, + expected_version: str, + ) -> None: + monkeypatch.setattr(litellm, "http2", use_http2) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", False) + + post_version, stream_version, body = _sync_exchange(http2_tls_server) + + assert post_version == expected_version + assert stream_version == expected_version + assert b"data: chunk-0" in body diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index f8868cfaf83..a52bf58e944 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1675,3 +1675,30 @@ async def test_bounded_get_closes_stream_on_cancellation(respx_mock, monkeypatch finally: await handler.close() assert closed.is_set() + + +@pytest.mark.asyncio +async def test_http2_flag_bypasses_aiohttp_transport(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + monkeypatch.setattr(litellm, "force_ipv4", False) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) + + monkeypatch.setattr(litellm, "http2", True) + assert AsyncHTTPHandler._should_use_aiohttp_transport() is False + assert AsyncHTTPHandler._create_async_transport() is None + + monkeypatch.setattr(litellm, "http2", False) + monkeypatch.setenv("LITELLM_HTTP2", "True") + assert AsyncHTTPHandler._should_use_aiohttp_transport() is False + assert AsyncHTTPHandler._create_async_transport() is None + + +@pytest.mark.asyncio +async def test_http2_disabled_by_default(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "http2", False) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + + assert AsyncHTTPHandler._should_use_aiohttp_transport() is True diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index d3c21c5bd5a..b54ec10ef17 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -411,3 +411,5 @@ async def test_async_genuine_bad_request_still_raises(provider, stream): ) def test_is_openai_backed_api_base_decides_by_hostname_only(api_base, expected): assert is_openai_backed_api_base(api_base) is expected + + diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 3dccb2b35bf..7fcdc8473d7 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -4,6 +4,7 @@ from datetime import datetime import contextlib import copy import json +import logging import os from collections.abc import Mapping from dataclasses import dataclass @@ -3850,3 +3851,27 @@ def test_bridged_responses_with_openai_http_handler_keeps_forwarded_headers_out_ assert "extra_headers" not in body assert body["model"] == "gpt-5.4" assert {k: request.headers[k] for k in FORWARDED_CLIENT_HEADERS} == FORWARDED_CLIENT_HEADERS + + +@pytest.mark.parametrize("http2_on", [True, False]) +def test_aiohttp_openai_warns_only_when_http2_enabled( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, http2_on: bool +): + from litellm.main import base_llm_aiohttp_handler + + monkeypatch.setattr(litellm, "http2", http2_on) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + + handler_completion: Final = MagicMock(return_value=MagicMock()) + monkeypatch.setattr(base_llm_aiohttp_handler, "completion", handler_completion) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + litellm.completion( + model="aiohttp_openai/gpt-4o", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-test", + ) + + assert handler_completion.called + warned: Final = "aiohttp_openai/ always uses aiohttp" in caplog.text + assert warned is http2_on diff --git a/uv.lock b/uv.lock index 08c4e4da76c..f8c7a0d7e83 100644 --- a/uv.lock +++ b/uv.lock @@ -3295,6 +3295,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + [[package]] name = "httpx-sse" version = "0.4.3" @@ -4464,7 +4469,7 @@ dependencies = [ { name = "boto3" }, { name = "click" }, { name = "fastuuid" }, - { name = "httpx" }, + { name = "httpx", extra = ["http2"] }, { name = "importlib-metadata" }, { name = "jinja2" }, { name = "jsonschema" }, @@ -4717,7 +4722,7 @@ requires-dist = [ { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" }, { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, { name = "hiredis", marker = "extra == 'proxy'", specifier = ">=3.0.0,<4.0" }, - { name = "httpx", specifier = ">=0.28.0,<1.0" }, + { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" },