diff --git a/litellm/__init__.py b/litellm/__init__.py index ccfbf80369f..5e4a76503cd 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -524,6 +524,7 @@ aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings disable_aiohttp_transport: bool = False # Set this to true to use httpx instead disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. +http2: bool = False # when True, LiteLLM-built httpx clients negotiate HTTP/2 over TLS (falls back to HTTP/1.1); bypasses aiohttp transport network_mock: bool = False # When True, use mock transport — no real network calls ####### STOP SEQUENCE LIMIT ####### diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index f4883b57fbc..cc37d1d9eca 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -74,6 +74,12 @@ _IPV4_LOCAL_ADDRESS: Final = "0.0.0.0" _HttpxTransportT = TypeVar("_HttpxTransportT", HTTPTransport, AsyncHTTPTransport) +def http2_enabled() -> bool: + from litellm.secret_managers.main import str_to_bool + + return litellm.http2 is True or str_to_bool(os.getenv("LITELLM_HTTP2", "False")) is True + + def _environment_proxy_mounts( build_proxy_transport: Callable[[str], _HttpxTransportT], ) -> Mapping[str, _HttpxTransportT | None]: @@ -638,6 +644,7 @@ class AsyncHTTPHandler: headers=default_headers, cookies=blocked_cookie_jar(), follow_redirects=True, + http2=http2_enabled(), ) async def close(self): @@ -1157,6 +1164,10 @@ class AsyncHTTPHandler: from litellm.secret_managers.main import str_to_bool + if http2_enabled(): + verbose_logger.debug("LITELLM_HTTP2 enabled, using httpx transport (aiohttp has no HTTP/2 support)") + return False + ######################################################### # Check if user disabled aiohttp transport ######################################################## @@ -1287,7 +1298,7 @@ class AsyncHTTPHandler: - [Default] If force_ipv4 is False, it will return None """ if litellm.force_ipv4: - return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) + return AsyncHTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled()) else: return None @@ -1300,7 +1311,7 @@ class AsyncHTTPHandler: if not isinstance(transport, AsyncHTTPTransport): return None return _environment_proxy_mounts( - lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert) + lambda proxy_url: AsyncHTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled()) ) @@ -1342,6 +1353,7 @@ class HTTPHandler: headers=default_headers, cookies=blocked_cookie_jar(), follow_redirects=True, + http2=http2_enabled(), ) @property @@ -1616,7 +1628,7 @@ class HTTPHandler: Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them """ if litellm.force_ipv4: - return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS) + return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, http2=http2_enabled()) else: return getattr(litellm, "sync_transport", None) @@ -1627,7 +1639,9 @@ class HTTPHandler: ) -> Mapping[str, HTTPTransport | None] | None: if not litellm.force_ipv4: return None - return _environment_proxy_mounts(lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert)) + return _environment_proxy_mounts( + lambda proxy_url: HTTPTransport(proxy=proxy_url, verify=verify, cert=cert, http2=http2_enabled()) + ) def get_async_httpx_client( diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 2db6d78a218..cb6a5e4e96a 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -32,6 +32,7 @@ from litellm.llms.custom_httpx.http_handler import ( _DEFAULT_TTL_FOR_HTTPX_CLIENTS, AsyncHTTPHandler, get_ssl_configuration, + http2_enabled, ) @@ -325,6 +326,7 @@ class BaseOpenAILLM: transport=transport, mounts=AsyncHTTPHandler._create_httpx_proxy_mounts(transport, verify=ssl_config, cert=None), follow_redirects=True, + http2=http2_enabled(), ) @staticmethod @@ -343,6 +345,7 @@ class BaseOpenAILLM: return httpx.Client( verify=ssl_config, follow_redirects=True, + http2=http2_enabled(), ) diff --git a/pyproject.toml b/pyproject.toml index 62ce4b4fd61..de6a6e9c9ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ # When changing a floor, verify it installs + imports on every supported # Python with: `uv pip install --resolution=lowest-direct .` "fastuuid>=0.14.0,<1.0", - "httpx>=0.28.0,<1.0", + "httpx[http2]>=0.28.0,<1.0", "openai>=2.20.0,<3.0.0", "python-dotenv>=1.0.0,<2.0", "tiktoken>=0.8.0,<1.0", diff --git a/tests/test_litellm/llms/conftest.py b/tests/test_litellm/llms/conftest.py new file mode 100644 index 00000000000..2905b606e51 --- /dev/null +++ b/tests/test_litellm/llms/conftest.py @@ -0,0 +1,124 @@ +import asyncio +import datetime +import ipaddress +import socket +import threading +import time + +import pytest + + +def _write_self_signed_cert(cert_dir): + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + now = datetime.datetime.now(datetime.timezone.utc) + cert = ( + x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(x509.oid.NameOID.COMMON_NAME, "localhost")])) + .issuer_name(x509.Name([x509.NameAttribute(x509.oid.NameOID.COMMON_NAME, "localhost")])) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=7)) + .add_extension( + x509.SubjectAlternativeName( + [x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))] + ), + critical=False, + ) + .sign(key, hashes.SHA256()) + ) + cert_file = cert_dir / "cert.pem" + key_file = cert_dir / "key.pem" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ) + ) + return cert_file, key_file + + +async def _asgi_app(scope, receive, send): + if scope["type"] != "http": + return + while True: + message = await receive() + if message["type"] == "http.request" and not message.get("more_body"): + break + if message["type"] == "http.disconnect": + return + if scope["path"] == "/stream": + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"text/event-stream")], + } + ) + for index in range(3): + await send( + { + "type": "http.response.body", + "body": f"data: chunk-{index}\n\n".encode(), + "more_body": True, + } + ) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + return + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"application/json")], + } + ) + await send({"type": "http.response.body", "body": b'{"ok": true}'}) + + +@pytest.fixture(scope="module") +def http2_tls_server(tmp_path_factory): + """Hypercorn TLS server on an ephemeral port that negotiates h2 or http/1.1 via ALPN.""" + from hypercorn.asyncio import serve + from hypercorn.config import Config + + cert_dir = tmp_path_factory.mktemp("h2certs") + cert_file, key_file = _write_self_signed_cert(cert_dir) + + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + + shutdown = threading.Event() + + def _serve() -> None: + loop = asyncio.new_event_loop() + config = Config() + config.bind = [f"127.0.0.1:{port}"] + config.certfile = str(cert_file) + config.keyfile = str(key_file) + config.alpn_protocols = ["h2", "http/1.1"] + loop.run_until_complete(serve(_asgi_app, config, shutdown_trigger=lambda: asyncio.to_thread(shutdown.wait))) + loop.close() + + thread = threading.Thread(target=_serve, daemon=True) + thread.start() + + for _ in range(100): + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + break + except OSError: + time.sleep(0.05) + else: + pytest.fail("hypercorn test server did not start") + + yield f"https://127.0.0.1:{port}" + + shutdown.set() + thread.join(timeout=10) diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py index f8868cfaf83..e170a7f7a78 100644 --- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py @@ -1675,3 +1675,71 @@ async def test_bounded_get_closes_stream_on_cancellation(respx_mock, monkeypatch finally: await handler.close() assert closed.is_set() + + +@pytest.mark.asyncio +async def test_http2_flag_bypasses_aiohttp_transport(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + monkeypatch.setattr(litellm, "force_ipv4", False) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) + + monkeypatch.setattr(litellm, "http2", True) + assert AsyncHTTPHandler._should_use_aiohttp_transport() is False + assert AsyncHTTPHandler._create_async_transport() is None + + monkeypatch.setattr(litellm, "http2", False) + monkeypatch.setenv("LITELLM_HTTP2", "True") + assert AsyncHTTPHandler._should_use_aiohttp_transport() is False + assert AsyncHTTPHandler._create_async_transport() is None + + +@pytest.mark.asyncio +async def test_http2_disabled_by_default(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "http2", False) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.delenv("DISABLE_AIOHTTP_TRANSPORT", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", False) + + assert AsyncHTTPHandler._should_use_aiohttp_transport() is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_http2", [True, False]) +@pytest.mark.parametrize("handler_kind", ["async", "sync"]) +async def test_http_version_negotiated_over_tls(monkeypatch, http2_tls_server, handler_kind, use_http2): + monkeypatch.setattr(litellm, "http2", use_http2) + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "force_ipv4", False) + + expected_version = "HTTP/2" if use_http2 else "HTTP/1.1" + + if handler_kind == "async": + handler = AsyncHTTPHandler(ssl_verify=False) + try: + response = await handler.post(f"{http2_tls_server}/echo", json={"ping": "pong"}) + assert response.status_code == 200 + assert response.http_version == expected_version + + stream_response = await handler.post(f"{http2_tls_server}/stream", stream=True) + assert stream_response.http_version == expected_version + chunks = [chunk async for chunk in stream_response.aiter_bytes()] + assert chunks + await stream_response.aclose() + finally: + await handler.close() + else: + handler = HTTPHandler(ssl_verify=False) + try: + response = handler.post(f"{http2_tls_server}/echo", json={"ping": "pong"}) + assert response.status_code == 200 + assert response.http_version == expected_version + + stream_response = handler.post(f"{http2_tls_server}/stream", stream=True) + assert stream_response.http_version == expected_version + chunks = list(stream_response.iter_bytes()) + assert chunks + stream_response.close() + finally: + handler.close() diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index d3c21c5bd5a..3dc37c21e7a 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -411,3 +411,20 @@ async def test_async_genuine_bad_request_still_raises(provider, stream): ) def test_is_openai_backed_api_base_decides_by_hostname_only(api_base, expected): assert is_openai_backed_api_base(api_base) is expected + + +def test_litellm_built_http_clients_negotiate_http2_only_when_enabled(monkeypatch): + monkeypatch.delenv("LITELLM_HTTP2", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + monkeypatch.setattr(litellm, "http2", False) + async_client = BaseOpenAILLM._get_async_http_client() + sync_client = BaseOpenAILLM._get_sync_http_client() + assert async_client is not None and async_client._transport._pool._http2 is False + assert sync_client is not None and sync_client._transport._pool._http2 is False + + monkeypatch.setattr(litellm, "http2", True) + async_client = BaseOpenAILLM._get_async_http_client() + sync_client = BaseOpenAILLM._get_sync_http_client() + assert async_client is not None and async_client._transport._pool._http2 is True + assert sync_client is not None and sync_client._transport._pool._http2 is True diff --git a/uv.lock b/uv.lock index eb4cdef76f1..4917dd2f5cb 100644 --- a/uv.lock +++ b/uv.lock @@ -3295,6 +3295,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + [[package]] name = "httpx-sse" version = "0.4.3" @@ -4365,7 +4370,7 @@ dependencies = [ { name = "boto3" }, { name = "click" }, { name = "fastuuid" }, - { name = "httpx" }, + { name = "httpx", extra = ["http2"] }, { name = "importlib-metadata" }, { name = "jinja2" }, { name = "jsonschema" }, @@ -4617,7 +4622,7 @@ requires-dist = [ { name = "grpcio", marker = "extra == 'proxy-runtime'", specifier = "==1.78.0" }, { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, { name = "hiredis", marker = "extra == 'proxy'", specifier = ">=3.0.0,<4.0" }, - { name = "httpx", specifier = ">=0.28.0,<1.0" }, + { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" },