scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

fix(http): address review on outbound HTTP/2

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Devin AI 2026-09-15 18:25:26 +00:00
parent a426df108a
commit fd2fb4c44e
7 changed files with 269 additions and 254 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/__init__.py
View file

@ -524,7 +524,7 @@ aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
http2: bool = False # when True, LiteLLM-built httpx clients negotiate HTTP/2 over TLS (falls back to HTTP/1.1); bypasses aiohttp transport
http2: bool = False
network_mock: bool = False # When True, use mock transport — no real network calls
####### STOP SEQUENCE LIMIT #######

6
litellm/main.py
View file

@ -105,7 +105,7 @@ from litellm.llms.base_llm.base_model_iterator import (
)
from litellm.llms.bedrock.common_utils import BedrockModelInfo
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.llms.vertex_ai.common_utils import (
@ -2341,6 +2341,10 @@ def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
def _complete_aiohttp_openai(
ctx: _CompletionDispatchContext,
) -> _CompletionDispatchResult:
if http2_enabled():
verbose_logger.warning(
"litellm.http2 is enabled but aiohttp_openai/ always uses aiohttp, which has no HTTP/2 client; this request stays on HTTP/1.1"
)
acompletion: Final = ctx.acompletion
api_base = ctx.api_base
api_key = ctx.api_key

208
tests/e2e/llm_translation/test_outbound_http2_e2e.py Normal file
View file

@ -0,0 +1,208 @@
"""Outbound HTTP/2 negotiation for LiteLLM-built httpx clients.
Spins up a local hypercorn TLS server that offers h2 and http/1.1 over ALPN and
drives the real AsyncHTTPHandler / HTTPHandler at it, so the negotiated protocol
on the wire is the assertion. No running proxy or provider credentials needed,
which is why these tests carry no `e2e` marker (same shape as the markerless
harness checks under tests/e2e/load/).
"""
from __future__ import annotations
import asyncio
import datetime
import ipaddress
import socket
import threading
import time
from collections.abc import Iterator
from pathlib import Path
from typing import Final, cast
import pytest
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from hypercorn.asyncio import (
serve, # pyright: ignore[reportUnknownVariableType] # hypercorn's serve signature passes through untyped worker hooks
)
from hypercorn.config import Config
from hypercorn.typing import (
ASGIReceiveCallable,
ASGISendCallable,
HTTPResponseBodyEvent,
HTTPResponseStartEvent,
Scope,
)
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
def _write_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]:
key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
now: Final = datetime.datetime.now(datetime.timezone.utc)
name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])
cert: Final = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - datetime.timedelta(days=1))
.not_valid_after(now + datetime.timedelta(days=7))
.add_extension(
x509.SubjectAlternativeName([x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]),
critical=False,
)
.sign(key, hashes.SHA256())
)
cert_file: Final = cert_dir / "cert.pem"
key_file: Final = cert_dir / "key.pem"
cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
key_file.write_bytes(
key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.TraditionalOpenSSL,
serialization.NoEncryption(),
)
)
return cert_file, key_file
async def _asgi_app(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None:
if scope["type"] != "http":
return
while True:
message = await receive()
if message["type"] == "http.disconnect":
return
if message["type"] == "http.request" and not message["more_body"]:
break
if scope["path"] == "/stream":
await send(
HTTPResponseStartEvent(
type="http.response.start", status=200, headers=[(b"content-type", b"text/event-stream")]
)
)
for index in range(3):
await send(
HTTPResponseBodyEvent(
type="http.response.body", body=f"data: chunk-{index}\n\n".encode(), more_body=True
)
)
await send(HTTPResponseBodyEvent(type="http.response.body", body=b"", more_body=False))
return
await send(
HTTPResponseStartEvent(type="http.response.start", status=200, headers=[(b"content-type", b"application/json")])
)
await send(HTTPResponseBodyEvent(type="http.response.body", body=b'{"ok": true}', more_body=False))
@pytest.fixture(scope="module")
def http2_tls_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]:
cert_dir: Final = tmp_path_factory.mktemp("h2certs")
cert_file, key_file = _write_self_signed_cert(cert_dir)
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port: Final = cast(int, sock.getsockname()[1])
shutdown: Final = threading.Event()
def _serve() -> None:
loop: Final = asyncio.new_event_loop()
config: Final = Config()
config.bind = [f"127.0.0.1:{port}"]
config.certfile = str(cert_file)
config.keyfile = str(key_file)
config.alpn_protocols = ["h2", "http/1.1"]
loop.run_until_complete(serve(_asgi_app, config, shutdown_trigger=lambda: asyncio.to_thread(shutdown.wait)))
loop.close()
thread: Final = threading.Thread(target=_serve, daemon=True)
thread.start()
for _ in range(100):
try:
with socket.create_connection(("127.0.0.1", port), timeout=0.2):
break
except OSError:
time.sleep(0.05)
else:
pytest.fail("hypercorn test server did not start")
yield f"https://127.0.0.1:{port}"
shutdown.set()
thread.join(timeout=10)
def _async_exchange(base_url: str) -> tuple[str, str, bytes]:
async def _run() -> tuple[str, str, bytes]:
handler: Final = AsyncHTTPHandler(ssl_verify=False)
try:
response: Final = await handler.client.post(f"{base_url}/echo", json={"ping": "pong"})
post_version: Final = response.http_version
async with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response:
stream_version: Final = stream_response.http_version
body: Final = b"".join([chunk async for chunk in stream_response.aiter_bytes()])
return post_version, stream_version, body
finally:
await handler.close()
return asyncio.run(_run())
def _sync_exchange(base_url: str) -> tuple[str, str, bytes]:
handler: Final = HTTPHandler(ssl_verify=False)
try:
response: Final = handler.client.post(f"{base_url}/echo", json={"ping": "pong"})
post_version: Final = response.http_version
with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response:
stream_version: Final = stream_response.http_version
body: Final = b"".join(stream_response.iter_bytes())
return post_version, stream_version, body
finally:
handler.close()
class TestOutboundHttp2:
@pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")])
def test_async_handler_negotiates_http2_only_when_enabled(
self,
monkeypatch: pytest.MonkeyPatch,
http2_tls_server: str,
use_http2: bool,
expected_version: str,
) -> None:
monkeypatch.setattr(litellm, "http2", use_http2)
monkeypatch.delenv("LITELLM_HTTP2", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "force_ipv4", False)
post_version, stream_version, body = _async_exchange(http2_tls_server)
assert post_version == expected_version
assert stream_version == expected_version
assert b"data: chunk-0" in body
@pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")])
def test_sync_handler_negotiates_http2_only_when_enabled(
self,
monkeypatch: pytest.MonkeyPatch,
http2_tls_server: str,
use_http2: bool,
expected_version: str,
) -> None:
monkeypatch.setattr(litellm, "http2", use_http2)
monkeypatch.delenv("LITELLM_HTTP2", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "force_ipv4", False)
post_version, stream_version, body = _sync_exchange(http2_tls_server)
assert post_version == expected_version
assert stream_version == expected_version
assert b"data: chunk-0" in body

124
tests/test_litellm/llms/conftest.py
View file

@ -1,124 +0,0 @@
import asyncio
import datetime
import ipaddress
import socket
import threading
import time
import pytest
def _write_self_signed_cert(cert_dir):
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
now = datetime.datetime.now(datetime.timezone.utc)
cert = (
x509.CertificateBuilder()
.subject_name(x509.Name([x509.NameAttribute(x509.oid.NameOID.COMMON_NAME, "localhost")]))
.issuer_name(x509.Name([x509.NameAttribute(x509.oid.NameOID.COMMON_NAME, "localhost")]))
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - datetime.timedelta(days=1))
.not_valid_after(now + datetime.timedelta(days=7))
.add_extension(
x509.SubjectAlternativeName(
[x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]
),
critical=False,
)
.sign(key, hashes.SHA256())
)
cert_file = cert_dir / "cert.pem"
key_file = cert_dir / "key.pem"
cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
key_file.write_bytes(
key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.TraditionalOpenSSL,
serialization.NoEncryption(),
)
)
return cert_file, key_file
async def _asgi_app(scope, receive, send):
if scope["type"] != "http":
return
while True:
message = await receive()
if message["type"] == "http.request" and not message.get("more_body"):
break
if message["type"] == "http.disconnect":
return
if scope["path"] == "/stream":
await send(
{
"type": "http.response.start",
"status": 200,
"headers": [(b"content-type", b"text/event-stream")],
}
)
for index in range(3):
await send(
{
"type": "http.response.body",
"body": f"data: chunk-{index}\n\n".encode(),
"more_body": True,
}
)
await send({"type": "http.response.body", "body": b"", "more_body": False})
return
await send(
{
"type": "http.response.start",
"status": 200,
"headers": [(b"content-type", b"application/json")],
}
)
await send({"type": "http.response.body", "body": b'{"ok": true}'})
@pytest.fixture(scope="module")
def http2_tls_server(tmp_path_factory):
"""Hypercorn TLS server on an ephemeral port that negotiates h2 or http/1.1 via ALPN."""
from hypercorn.asyncio import serve
from hypercorn.config import Config
cert_dir = tmp_path_factory.mktemp("h2certs")
cert_file, key_file = _write_self_signed_cert(cert_dir)
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
shutdown = threading.Event()
def _serve() -> None:
loop = asyncio.new_event_loop()
config = Config()
config.bind = [f"127.0.0.1:{port}"]
config.certfile = str(cert_file)
config.keyfile = str(key_file)
config.alpn_protocols = ["h2", "http/1.1"]
loop.run_until_complete(serve(_asgi_app, config, shutdown_trigger=lambda: asyncio.to_thread(shutdown.wait)))
loop.close()
thread = threading.Thread(target=_serve, daemon=True)
thread.start()
for _ in range(100):
try:
with socket.create_connection(("127.0.0.1", port), timeout=0.2):
break
except OSError:
time.sleep(0.05)
else:
pytest.fail("hypercorn test server did not start")
yield f"https://127.0.0.1:{port}"
shutdown.set()
thread.join(timeout=10)

41
tests/test_litellm/llms/custom_httpx/test_http_handler.py
View file

@ -1702,44 +1702,3 @@ async def test_http2_disabled_by_default(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", False)
assert AsyncHTTPHandler._should_use_aiohttp_transport() is True
@pytest.mark.asyncio
@pytest.mark.parametrize("use_http2", [True, False])
@pytest.mark.parametrize("handler_kind", ["async", "sync"])
async def test_http_version_negotiated_over_tls(monkeypatch, http2_tls_server, handler_kind, use_http2):
monkeypatch.setattr(litellm, "http2", use_http2)
monkeypatch.delenv("LITELLM_HTTP2", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "force_ipv4", False)
expected_version = "HTTP/2" if use_http2 else "HTTP/1.1"
if handler_kind == "async":
handler = AsyncHTTPHandler(ssl_verify=False)
try:
response = await handler.post(f"{http2_tls_server}/echo", json={"ping": "pong"})
assert response.status_code == 200
assert response.http_version == expected_version
stream_response = await handler.post(f"{http2_tls_server}/stream", stream=True)
assert stream_response.http_version == expected_version
chunks = [chunk async for chunk in stream_response.aiter_bytes()]
assert chunks
await stream_response.aclose()
finally:
await handler.close()
else:
handler = HTTPHandler(ssl_verify=False)
try:
response = handler.post(f"{http2_tls_server}/echo", json={"ping": "pong"})
assert response.status_code == 200
assert response.http_version == expected_version
stream_response = handler.post(f"{http2_tls_server}/stream", stream=True)
assert stream_response.http_version == expected_version
chunks = list(stream_response.iter_bytes())
assert chunks
stream_response.close()
finally:
handler.close()

15
tests/test_litellm/llms/openai/test_openai_common_utils.py
View file

@ -413,18 +413,3 @@ def test_is_openai_backed_api_base_decides_by_hostname_only(api_base, expected):
assert is_openai_backed_api_base(api_base) is expected
def test_litellm_built_http_clients_negotiate_http2_only_when_enabled(monkeypatch):
monkeypatch.delenv("LITELLM_HTTP2", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "http2", False)
async_client = BaseOpenAILLM._get_async_http_client()
sync_client = BaseOpenAILLM._get_sync_http_client()
assert async_client is not None and async_client._transport._pool._http2 is False
assert sync_client is not None and sync_client._transport._pool._http2 is False
monkeypatch.setattr(litellm, "http2", True)
async_client = BaseOpenAILLM._get_async_http_client()
sync_client = BaseOpenAILLM._get_sync_http_client()
assert async_client is not None and async_client._transport._pool._http2 is True
assert sync_client is not None and sync_client._transport._pool._http2 is True

127
tests/test_litellm/test_main.py
View file

@ -190,9 +190,7 @@ async def test_url_with_format_param(model, sync_mode, monkeypatch):
# URL->image conversion helpers so suite-level network/client state from
# earlier tests cannot prevent the mocked provider client from being hit.
fake_base64_image = "data:image/png;base64,ZmFrZS1pbWFnZQ=="
monkeypatch.setattr(
prompt_factory, "convert_url_to_base64", lambda url: fake_base64_image
)
monkeypatch.setattr(prompt_factory, "convert_url_to_base64", lambda url: fake_base64_image)
monkeypatch.setattr(
prompt_factory.BedrockImageProcessor,
"get_image_details",
@ -307,9 +305,7 @@ async def test_url_with_format_param_openai(model, sync_mode):
}
],
}
with patch.object(
client.chat.completions.with_raw_response, "create"
) as mock_client:
with patch.object(client.chat.completions.with_raw_response, "create") as mock_client:
try:
if sync_mode:
response = completion(**args, client=client)
@ -361,9 +357,7 @@ def test_strip_input_examples_for_non_anthropic_providers():
}
]
assert not litellm_main._should_allow_input_examples(
custom_llm_provider="openai", model="gpt-4o-mini"
)
assert not litellm_main._should_allow_input_examples(custom_llm_provider="openai", model="gpt-4o-mini")
cleaned = litellm_main._drop_input_examples_from_tools(tools=tools)
@ -375,9 +369,7 @@ def test_strip_input_examples_for_non_anthropic_providers():
def test_custom_provider_with_extra_headers():
from litellm.llms.custom_httpx.http_handler import HTTPHandler
with patch.object(
litellm.llms.custom_httpx.http_handler.HTTPHandler, "post"
) as mock_post:
with patch.object(litellm.llms.custom_httpx.http_handler.HTTPHandler, "post") as mock_post:
response = litellm.completion(
model="custom/custom",
messages=[{"role": "user", "content": "Hello, how are you?"}],
@ -392,9 +384,7 @@ def test_custom_provider_with_extra_headers():
def test_custom_provider_with_extra_body():
from litellm.llms.custom_httpx.http_handler import HTTPHandler
with patch.object(
litellm.llms.custom_httpx.http_handler.HTTPHandler, "post"
) as mock_post:
with patch.object(litellm.llms.custom_httpx.http_handler.HTTPHandler, "post") as mock_post:
response = litellm.completion(
model="custom/custom",
messages=[{"role": "user", "content": "Hello, how are you?"}],
@ -421,9 +411,7 @@ def test_custom_provider_with_extra_body():
}
# test that extra_body is not passed if not provided
with patch.object(
litellm.llms.custom_httpx.http_handler.HTTPHandler, "post"
) as mock_post:
with patch.object(litellm.llms.custom_httpx.http_handler.HTTPHandler, "post") as mock_post:
response = litellm.completion(
model="custom/custom",
messages=[{"role": "user", "content": "Hello, how are you?"}],
@ -454,9 +442,7 @@ def set_openrouter_api_key():
@pytest.mark.asyncio
async def test_extra_body_with_fallback(
respx_mock: respx.MockRouter, set_openrouter_api_key, monkeypatch
):
async def test_extra_body_with_fallback(respx_mock: respx.MockRouter, set_openrouter_api_key, monkeypatch):
"""
test regression for https://github.com/BerriAI/litellm/issues/8425.
@ -524,9 +510,7 @@ async def test_extra_body_with_fallback(
# Verify the response
assert response is not None
assert (
len(respx_mock.calls) > 0
), "Mock was not called - check if aiohttp transport is properly disabled"
assert len(respx_mock.calls) > 0, "Mock was not called - check if aiohttp transport is properly disabled"
# Get the request from the mock
request: httpx.Request = respx_mock.calls[0].request
@ -550,9 +534,7 @@ async def test_extra_body_with_fallback(
@pytest.mark.parametrize("env_base", ["OPENAI_BASE_URL", "OPENAI_API_BASE"])
@pytest.mark.asyncio
@pytest.mark.flaky(retries=3, delay=1)
async def test_openai_env_base(
respx_mock: respx.MockRouter, env_base, openai_api_response, monkeypatch
):
async def test_openai_env_base(respx_mock: respx.MockRouter, env_base, openai_api_response, monkeypatch):
"This tests OpenAI env variables are honored, including legacy OPENAI_API_BASE"
# Ensure aiohttp transport is disabled to use httpx which respx can mock
litellm.disable_aiohttp_transport = True
@ -567,9 +549,7 @@ async def test_openai_env_base(
messages = [{"role": "user", "content": "Hello, how are you?"}]
# Configure respx mock to intercept the request
mock_route = respx_mock.post(
url__regex=r"http://localhost:12345/v1/chat/completions.*"
).mock(
mock_route = respx_mock.post(url__regex=r"http://localhost:12345/v1/chat/completions.*").mock(
return_value=httpx.Response(
status_code=200,
json={
@ -603,9 +583,7 @@ async def test_openai_env_base(
assert response.choices[0].message.content == "Hello from mocked response!"
# Verify the mock was called
assert (
mock_route.called
), "Mock route was not called - request may have bypassed respx"
assert mock_route.called, "Mock route was not called - request may have bypassed respx"
finally:
# Clean up to avoid affecting other tests
litellm.disable_aiohttp_transport = False
@ -681,9 +659,7 @@ def test_completion_forwards_verbosity_in_raw_request(respx_mock: respx.MockRout
model = "gpt-5.2"
messages = [{"role": "user", "content": "hi"}]
respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
return_value=_mocked_openai_chat_response(model)
)
respx_mock.post("https://api.openai.com/v1/chat/completions").mock(return_value=_mocked_openai_chat_response(model))
request = return_raw_request(
endpoint=CallTypes.completion,
@ -700,9 +676,7 @@ def test_completion_forwards_verbosity_in_raw_request(respx_mock: respx.MockRout
@pytest.mark.asyncio
async def test_acompletion_forwards_verbosity_to_provider_request(
respx_mock: respx.MockRouter, monkeypatch
):
async def test_acompletion_forwards_verbosity_to_provider_request(respx_mock: respx.MockRouter, monkeypatch):
"""Regression test: acompletion() must forward the verbosity param to the provider request body."""
original_disable_aiohttp = litellm.disable_aiohttp_transport
try:
@ -763,9 +737,9 @@ def test_responses_api_bridge_check_gpt_5_4_pro():
model=model_name,
custom_llm_provider="openai",
)
assert (
model_info.get("mode") == "responses"
), f"{model_name} should have mode='responses', got '{model_info.get('mode')}'"
assert model_info.get("mode") == "responses", (
f"{model_name} should have mode='responses', got '{model_info.get('mode')}'"
)
def test_responses_api_bridge_check_gpt_5_4_tools_plus_reasoning_routes_to_responses():
@ -1182,7 +1156,7 @@ def test_responses_api_bridge_check_openai_backed_custom_api_base_with_unset_eff
tools=[{"type": "function", "function": {"name": "get_capital"}}],
reasoning_effort=None,
api_base=api_base,
)
)
assert model == "gpt-5.6"
assert model_info.get("mode") == "responses"
@ -1207,7 +1181,7 @@ def test_responses_api_bridge_check_lookalike_custom_api_base_with_unset_effort_
tools=[{"type": "function", "function": {"name": "get_capital"}}],
reasoning_effort=None,
api_base=api_base,
)
)
assert model == "gpt-5.6"
assert model_info.get("mode") != "responses"
@ -1227,7 +1201,7 @@ def test_responses_api_bridge_check_privatelink_api_base_via_env_with_unset_effo
tools=[{"type": "function", "function": {"name": "get_capital"}}],
reasoning_effort=None,
api_base=None,
)
)
assert model == "gpt-5.6"
assert model_info.get("mode") == "responses"
@ -1588,9 +1562,7 @@ def test_responses_api_bridge_check_handles_exception():
with patch("litellm.main._get_model_info_helper") as mock_get_model_info:
mock_get_model_info.side_effect = Exception("Model not found")
model_info, model = responses_api_bridge_check(
model="responses/custom-model", custom_llm_provider="custom"
)
model_info, model = responses_api_bridge_check(model="responses/custom-model", custom_llm_provider="custom")
assert model == "custom-model"
assert model_info["mode"] == "responses"
@ -2371,9 +2343,7 @@ def test_image_edit_merges_headers_and_extra_headers():
mock_image_edit_config = MagicMock()
mock_image_edit_config.get_supported_openai_params.return_value = set()
mock_image_edit_config.map_openai_params.side_effect = lambda **kwargs: dict(
kwargs["image_edit_optional_params"]
)
mock_image_edit_config.map_openai_params.side_effect = lambda **kwargs: dict(kwargs["image_edit_optional_params"])
with (
patch(
@ -2729,10 +2699,7 @@ def test_mock_completion_stream_with_model_response():
# Verify the content is streamed correctly
accumulated_content = ""
for chunk in chunks:
if (
hasattr(chunk.choices[0].delta, "content")
and chunk.choices[0].delta.content
):
if hasattr(chunk.choices[0].delta, "content") and chunk.choices[0].delta.content:
accumulated_content += chunk.choices[0].delta.content
assert "This is a test response" in accumulated_content or len(chunks) > 0
@ -2790,10 +2757,7 @@ async def test_async_mock_completion_stream_with_model_response():
# Verify the content is streamed correctly
accumulated_content = ""
for chunk in chunks:
if (
hasattr(chunk.choices[0].delta, "content")
and chunk.choices[0].delta.content
):
if hasattr(chunk.choices[0].delta, "content") and chunk.choices[0].delta.content:
accumulated_content += chunk.choices[0].delta.content
assert "This is an async test response" in accumulated_content or len(chunks) > 0
@ -2860,9 +2824,7 @@ def test_stream_chunk_builder_text_completion_combines_text_and_usage():
),
]
response = stream_chunk_builder_text_completion(
chunks=chunks, messages=[{"role": "user", "content": "say hello"}]
)
response = stream_chunk_builder_text_completion(chunks=chunks, messages=[{"role": "user", "content": "say hello"}])
assert response.choices[0].text == "Hello world"
assert response.choices[0].finish_reason == "stop"
@ -3310,10 +3272,7 @@ def _text_chunk(content, finish_reason=None, usage=None):
def _priced_at(prompt_tokens, completion_tokens):
prices = litellm.model_cost[STREAM_COST_MODEL]
return (
prompt_tokens * prices["input_cost_per_token"]
+ completion_tokens * prices["output_cost_per_token"]
)
return prompt_tokens * prices["input_cost_per_token"] + completion_tokens * prices["output_cost_per_token"]
@pytest.fixture
@ -3380,9 +3339,9 @@ def test_streaming_and_not_streaming_bill_the_same_usage_the_same(local_cost_map
usage=STREAMED_USAGE,
)
assert litellm.completion_cost(
completion_response=rebuilt, model=STREAM_COST_MODEL
) == pytest.approx(litellm.completion_cost(completion_response=whole, model=STREAM_COST_MODEL))
assert litellm.completion_cost(completion_response=rebuilt, model=STREAM_COST_MODEL) == pytest.approx(
litellm.completion_cost(completion_response=whole, model=STREAM_COST_MODEL)
)
def test_a_stream_that_reported_no_usage_is_still_billed(local_cost_map):
@ -3401,9 +3360,7 @@ def test_a_stream_that_reported_no_usage_is_still_billed(local_cost_map):
cost = litellm.completion_cost(completion_response=rebuilt, model=STREAM_COST_MODEL)
assert cost > 0
assert cost == pytest.approx(
_priced_at(rebuilt.usage.prompt_tokens, rebuilt.usage.completion_tokens)
)
assert cost == pytest.approx(_priced_at(rebuilt.usage.prompt_tokens, rebuilt.usage.completion_tokens))
@pytest.mark.asyncio
@ -3850,3 +3807,29 @@ def test_bridged_responses_with_openai_http_handler_keeps_forwarded_headers_out_
assert "extra_headers" not in body
assert body["model"] == "gpt-5.4"
assert {k: request.headers[k] for k in FORWARDED_CLIENT_HEADERS} == FORWARDED_CLIENT_HEADERS
@pytest.mark.parametrize("http2_on", [True, False])
def test_aiohttp_openai_warns_only_when_http2_enabled(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, http2_on: bool
):
import logging
from litellm.main import base_llm_aiohttp_handler
monkeypatch.setattr(litellm, "http2", http2_on)
monkeypatch.delenv("LITELLM_HTTP2", raising=False)
handler_completion: Final = MagicMock(return_value=MagicMock())
monkeypatch.setattr(base_llm_aiohttp_handler, "completion", handler_completion)
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
litellm.completion(
model="aiohttp_openai/gpt-4o",
messages=[{"role": "user", "content": "hi"}],
api_key="sk-test",
)
assert handler_completion.called
warned: Final = "aiohttp_openai/ always uses aiohttp" in caplog.text
assert warned is http2_on