mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(http2): keep TE, force_ipv4 trust, evicted-client and opt-out behavior on the HTTP/2 path
- drop TE values other than trailers on https requests through the HTTP/2 spread, since h2 refuses them - force_ipv4 transports keep httpx's default trust store (SSL_CERT_DIR) unless SSL is configured - the evicted client closer asks the HTTP/2 spread whether a request is in flight - LITELLM_HTTP2 set to 0, no or off, and a quoted "false" http2 setting, opt out of HTTP/2 - the wire test's held-search barrier waits on an event instead of polling
This commit is contained in:
parent
939b86f6ae
commit
4401833140
5 changed files with 133 additions and 27 deletions
|
|
@ -139,6 +139,9 @@ def _has_connection_in_flight(client: object) -> bool:
|
|||
if getattr(getattr(client, "connection_pool", None), "_in_use_connections", None):
|
||||
return True
|
||||
transport: Final = _transport_of(client)
|
||||
spread_in_flight: Final[object] = getattr(transport, "has_request_in_flight", None)
|
||||
if callable(spread_in_flight):
|
||||
return bool(spread_in_flight())
|
||||
pooled_busy: Final = _pool_has_busy_connection(transport)
|
||||
if pooled_busy is not None:
|
||||
return pooled_busy
|
||||
|
|
|
|||
|
|
@ -74,6 +74,21 @@ class _AsyncReleasingStream(httpx.AsyncByteStream):
|
|||
await self._stream.aclose()
|
||||
|
||||
|
||||
def _allowed_over_http2(name: bytes, value: bytes) -> bool:
|
||||
return name.lower() != b"te" or value.strip().lower() == b"trailers"
|
||||
|
||||
|
||||
def _without_te_codings(request: httpx.Request) -> httpx.Request:
|
||||
if request.url.scheme != "https":
|
||||
return request
|
||||
kept: Final = tuple((name, value) for name, value in request.headers.raw if _allowed_over_http2(name, value))
|
||||
if len(kept) == len(request.headers.raw):
|
||||
return request
|
||||
return httpx.Request(
|
||||
request.method, request.url, headers=kept, stream=request.stream, extensions=request.extensions
|
||||
)
|
||||
|
||||
|
||||
def _with_stream(response: httpx.Response, stream: httpx.AsyncByteStream) -> httpx.Response:
|
||||
return httpx.Response(response.status_code, headers=response.headers, stream=stream, extensions=response.extensions)
|
||||
|
||||
|
|
@ -90,7 +105,7 @@ class AsyncHTTP2ConnectionSpread(httpx.AsyncBaseTransport):
|
|||
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
||||
lane: Final = self._lanes.acquire()
|
||||
try:
|
||||
response: Final = await lane.transport.handle_async_request(request)
|
||||
response: Final = await lane.transport.handle_async_request(_without_te_codings(request))
|
||||
except BaseException:
|
||||
lane.release()
|
||||
raise
|
||||
|
|
@ -100,6 +115,9 @@ class AsyncHTTP2ConnectionSpread(httpx.AsyncBaseTransport):
|
|||
return response
|
||||
return _with_stream(response, _AsyncReleasingStream(stream, lane.release))
|
||||
|
||||
def has_request_in_flight(self) -> bool:
|
||||
return any(lane.in_flight > 0 for lane in self._lanes.lanes)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
for lane in self._lanes.lanes:
|
||||
await lane.transport.aclose()
|
||||
|
|
|
|||
|
|
@ -86,10 +86,15 @@ def http2_enabled() -> bool:
|
|||
return litellm.http2 is True or str_to_bool(os.getenv("LITELLM_HTTP2", "False")) is True
|
||||
|
||||
|
||||
def http2_disabled() -> bool:
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
_HTTP2_OFF_VALUES: Final = frozenset({"false", "0", "no", "off"})
|
||||
|
||||
return litellm.http2 is False or str_to_bool(os.getenv("LITELLM_HTTP2")) is False
|
||||
|
||||
def _is_http2_off(value: bool | str | None) -> bool:
|
||||
return value is False or (isinstance(value, str) and value.strip().lower() in _HTTP2_OFF_VALUES)
|
||||
|
||||
|
||||
def http2_disabled() -> bool:
|
||||
return _is_http2_off(litellm.http2) or _is_http2_off(os.getenv("LITELLM_HTTP2"))
|
||||
|
||||
|
||||
def _resolve_http2(provider_default: bool, ssl_verify: VerifyTypes | None) -> bool:
|
||||
|
|
@ -467,6 +472,11 @@ def get_ssl_configuration(
|
|||
return ssl_verify
|
||||
|
||||
|
||||
def _httpx_default_verify_unless_configured(verify: VerifyTypes) -> VerifyTypes:
|
||||
litellm_defaults: Final = (get_ssl_configuration(True), get_ssl_configuration(True, http2=True))
|
||||
return True if any(verify is default for default in litellm_defaults) else verify
|
||||
|
||||
|
||||
_shared_realtime_ssl_context: bool | str | ssl.SSLContext | None = None
|
||||
|
||||
|
||||
|
|
@ -1438,15 +1448,20 @@ class AsyncHTTPHandler:
|
|||
- [Default] Otherwise it will return None
|
||||
"""
|
||||
local_address: Final = _IPV4_LOCAL_ADDRESS if litellm.force_ipv4 else None
|
||||
transport_verify: Final = (
|
||||
_httpx_default_verify_unless_configured(verify)
|
||||
if litellm.force_ipv4 and not AsyncHTTPHandler.should_use_aiohttp_transport()
|
||||
else verify
|
||||
)
|
||||
if http2 or http2_enabled():
|
||||
return AsyncHTTP2ConnectionSpread(
|
||||
lambda lane_limits: AsyncHTTPTransport(
|
||||
local_address=local_address, verify=verify, cert=cert, http2=True, limits=lane_limits
|
||||
local_address=local_address, verify=transport_verify, cert=cert, http2=True, limits=lane_limits
|
||||
),
|
||||
limits,
|
||||
)
|
||||
if litellm.force_ipv4:
|
||||
return AsyncHTTPTransport(local_address=local_address, verify=verify, cert=cert, limits=limits)
|
||||
return AsyncHTTPTransport(local_address=local_address, verify=transport_verify, cert=cert, limits=limits)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -1790,7 +1805,12 @@ class HTTPHandler:
|
|||
Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
|
||||
"""
|
||||
if litellm.force_ipv4:
|
||||
return HTTPTransport(local_address=_IPV4_LOCAL_ADDRESS, verify=verify, cert=cert, http2=http2)
|
||||
return HTTPTransport(
|
||||
local_address=_IPV4_LOCAL_ADDRESS,
|
||||
verify=_httpx_default_verify_unless_configured(verify),
|
||||
cert=cert,
|
||||
http2=http2,
|
||||
)
|
||||
return getattr(litellm, "sync_transport", None)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import json
|
|||
import queue
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Generator, Iterator
|
||||
|
|
@ -40,6 +41,7 @@ from integration._support.vertex import service_account_json
|
|||
STREAM_CHUNKS: Final = 3
|
||||
HELD_SEARCHES: Final = 150
|
||||
HOLD_SECONDS: Final = 10.0
|
||||
PROBED_HEADERS: Final = frozenset({b"te", b"keep-alive", b"connection", b"proxy-connection", b"upgrade", b"x-probe"})
|
||||
VERTEX_SERVING_CONFIG_PATH: Final = (
|
||||
"/v1/projects/p/locations/global/collections/default_collection/engines/e/servingConfigs/default_search"
|
||||
)
|
||||
|
|
@ -48,6 +50,7 @@ VERTEX_SERVING_CONFIG_PATH: Final = (
|
|||
@dataclass(frozen=True, slots=True)
|
||||
class PeerLog:
|
||||
held_arrivals: queue.SimpleQueue[int] = field(default_factory=queue.SimpleQueue)
|
||||
all_held_arrived: asyncio.Event = field(default_factory=asyncio.Event)
|
||||
token_requests: queue.SimpleQueue[str] = field(default_factory=queue.SimpleQueue)
|
||||
counted_client_ports: queue.SimpleQueue[int] = field(default_factory=queue.SimpleQueue)
|
||||
|
||||
|
|
@ -122,6 +125,10 @@ async def _peer(log: PeerLog, scope: Scope, receive: ASGIReceiveCallable, send:
|
|||
log.counted_client_ports.put(counted_client[1] if counted_client else 0)
|
||||
await _send_json(send, {"results": [{"id": "counted"}]})
|
||||
return
|
||||
if scope["path"].startswith("/headers/") and scope["path"].endswith(":search"):
|
||||
seen: Final = sorted(name.decode() for name, _ in scope["headers"] if name in PROBED_HEADERS)
|
||||
await _send_json(send, {"results": [{"id": f"http/{version}"}, *({"id": name} for name in seen)]})
|
||||
return
|
||||
if scope["path"].endswith(":search"):
|
||||
await _send_json(send, {"results": [{"id": f"http/{version}"}]})
|
||||
return
|
||||
|
|
@ -144,10 +151,12 @@ async def _peer(log: PeerLog, scope: Scope, receive: ASGIReceiveCallable, send:
|
|||
|
||||
async def _hold_until_all_arrive(log: PeerLog, client_port: int) -> str:
|
||||
log.held_arrivals.put(client_port)
|
||||
deadline: Final = time.monotonic() + HOLD_SECONDS
|
||||
while log.held_arrivals.qsize() < HELD_SEARCHES and time.monotonic() < deadline:
|
||||
await asyncio.sleep(0.01)
|
||||
return "held-together" if log.held_arrivals.qsize() >= HELD_SEARCHES else "released-alone"
|
||||
if log.held_arrivals.qsize() >= HELD_SEARCHES:
|
||||
log.all_held_arrived.set()
|
||||
waiter: Final = asyncio.ensure_future(log.all_held_arrived.wait())
|
||||
await asyncio.wait((waiter,), timeout=HOLD_SECONDS)
|
||||
waiter.cancel()
|
||||
return "held-together" if log.all_held_arrived.is_set() else "released-alone"
|
||||
|
||||
|
||||
async def _send_json(send: ASGISendCallable, body: dict[str, object]) -> None:
|
||||
|
|
@ -191,11 +200,11 @@ def http2_tls_peer(tmp_path_factory: pytest.TempPathFactory) -> Iterator[TlsPeer
|
|||
thread.join(timeout=10)
|
||||
|
||||
|
||||
def _async_exchange(base_url: str) -> Observed:
|
||||
def _async_exchange(base_url: str, ssl_verify: bool | None = False) -> Observed:
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
async def _run() -> Observed:
|
||||
handler: Final = AsyncHTTPHandler(ssl_verify=False)
|
||||
handler: Final = AsyncHTTPHandler(ssl_verify=ssl_verify)
|
||||
try:
|
||||
response: Final = await handler.client.post(f"{base_url}/echo", json={"ping": "pong"})
|
||||
async with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response:
|
||||
|
|
@ -211,10 +220,10 @@ def _async_exchange(base_url: str) -> Observed:
|
|||
return asyncio.run(_run())
|
||||
|
||||
|
||||
def _sync_exchange(base_url: str) -> Observed:
|
||||
def _sync_exchange(base_url: str, ssl_verify: bool | None = False) -> Observed:
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
handler: Final = HTTPHandler(ssl_verify=False)
|
||||
handler: Final = HTTPHandler(ssl_verify=ssl_verify)
|
||||
try:
|
||||
response: Final = handler.client.post(f"{base_url}/echo", json={"ping": "pong"})
|
||||
with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response:
|
||||
|
|
@ -292,7 +301,10 @@ def _vertex_search_file_ids(
|
|||
|
||||
|
||||
async def _avertex_search_file_ids(
|
||||
peer: TlsPeer, search_url: str | None = None, ssl_verify: ssl.SSLContext | None = None
|
||||
peer: TlsPeer,
|
||||
search_url: str | None = None,
|
||||
ssl_verify: ssl.SSLContext | None = None,
|
||||
extra_headers: dict[str, object] | None = None,
|
||||
) -> list[str | None]:
|
||||
import litellm
|
||||
|
||||
|
|
@ -304,6 +316,7 @@ async def _avertex_search_file_ids(
|
|||
vertex_credentials=service_account_json("p", peer.url),
|
||||
vertex_project="p",
|
||||
ssl_verify=ssl_verify,
|
||||
extra_headers=extra_headers,
|
||||
)
|
||||
return [result.get("file_id") for result in response.get("data") or ()]
|
||||
|
||||
|
|
@ -383,21 +396,48 @@ def test_vertex_search_over_forced_ipv4_trusts_the_configured_ca(
|
|||
assert asyncio.run(_avertex_search_file_ids(http2_tls_peer)) == ["http/2"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("off", ["False", "0", "no", " OFF "])
|
||||
def test_explicit_http2_off_keeps_vertex_search_on_http1(
|
||||
monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer
|
||||
monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, off: str
|
||||
) -> None:
|
||||
_trust_peer_without_global_http2(monkeypatch, http2_tls_peer)
|
||||
monkeypatch.setenv("LITELLM_HTTP2", off)
|
||||
assert asyncio.run(_avertex_search_file_ids(http2_tls_peer)) == ["http/1.1"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("off", [False, "false"])
|
||||
def test_http2_setting_off_keeps_vertex_search_on_http1(
|
||||
monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, off: bool | str
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
_trust_peer_without_global_http2(monkeypatch, http2_tls_peer)
|
||||
monkeypatch.setenv("LITELLM_HTTP2", "False")
|
||||
assert _vertex_search_file_ids(http2_tls_peer) == ["http/1.1"]
|
||||
monkeypatch.setattr(litellm, "http2", off)
|
||||
assert asyncio.run(_avertex_search_file_ids(http2_tls_peer)) == ["http/1.1"]
|
||||
|
||||
monkeypatch.delenv("LITELLM_HTTP2")
|
||||
monkeypatch.setattr(litellm, "http2", False)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
assert _vertex_search_file_ids(http2_tls_peer) == ["http/1.1"]
|
||||
assert asyncio.run(_avertex_search_file_ids(http2_tls_peer)) == ["http/1.1"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("te", "forwarded"), [("gzip", ["http/2", "x-probe"]), ("trailers", ["http/2", "te", "x-probe"])]
|
||||
)
|
||||
def test_async_vertex_search_over_http2_sends_no_connection_specific_headers(
|
||||
monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, te: str, forwarded: list[str]
|
||||
) -> None:
|
||||
_trust_peer_without_global_http2(monkeypatch, http2_tls_peer)
|
||||
received: Final = asyncio.run(
|
||||
_avertex_search_file_ids(
|
||||
http2_tls_peer,
|
||||
search_url=f"{http2_tls_peer.url}/headers",
|
||||
extra_headers={
|
||||
"TE": te,
|
||||
"Keep-Alive": "timeout=5",
|
||||
"Connection": "keep-alive",
|
||||
"Proxy-Connection": "keep-alive",
|
||||
"Upgrade": "h2c",
|
||||
"X-Probe": "kept",
|
||||
},
|
||||
)
|
||||
)
|
||||
assert received == forwarded
|
||||
|
||||
|
||||
def _alpn_offered_by(context: ssl.SSLContext, peer: TlsPeer) -> str | None:
|
||||
|
|
@ -665,6 +705,30 @@ def test_forced_ipv4_handlers_honour_ssl_verify_false(monkeypatch: pytest.Monkey
|
|||
_assert_negotiated(_sync_exchange(http2_tls_peer.url), enabled)
|
||||
|
||||
|
||||
def _ca_dir_holding(cert_file: Path, ca_dir: Path) -> Path:
|
||||
subject_hash: Final = subprocess.run(
|
||||
["openssl", "x509", "-hash", "-noout", "-in", str(cert_file)], capture_output=True, text=True, check=True
|
||||
).stdout.strip()
|
||||
(ca_dir / f"{subject_hash}.0").write_bytes(cert_file.read_bytes())
|
||||
return ca_dir
|
||||
|
||||
|
||||
def test_forced_ipv4_handlers_trust_a_ca_found_only_in_ssl_cert_dir(
|
||||
monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, tmp_path: Path
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "force_ipv4", True)
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.delenv("SSL_VERIFY", raising=False)
|
||||
monkeypatch.delenv("SSL_CERT_FILE", raising=False)
|
||||
monkeypatch.setenv("SSL_CERT_DIR", str(_ca_dir_holding(http2_tls_peer.cert_file, tmp_path)))
|
||||
for enabled in (False, True):
|
||||
_set_http2(monkeypatch, enabled)
|
||||
_assert_negotiated(_async_exchange(http2_tls_peer.url, ssl_verify=None), enabled)
|
||||
_assert_negotiated(_sync_exchange(http2_tls_peer.url, ssl_verify=None), enabled)
|
||||
|
||||
|
||||
def test_vertex_search_over_forced_ipv4_presents_the_client_certificate(
|
||||
monkeypatch: pytest.MonkeyPatch, http2_tls_peer: TlsPeer, tmp_path: Path
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -356,13 +356,14 @@ async def test_a_client_serving_a_request_is_not_closed_when_its_grace_window_en
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_aiohttp_backed_handler_is_not_closed_mid_request():
|
||||
"""The default async path is aiohttp-backed, whose pool accounts for its own leases."""
|
||||
@pytest.mark.parametrize("http2", [False, True], ids=["aiohttp", "http2-connection-spread"])
|
||||
async def test_a_handler_is_not_closed_mid_request(http2: bool):
|
||||
"""Both async handler paths account for their own leases: aiohttp's pool and the HTTP/2 connection spread."""
|
||||
server = await asyncio.start_server(_trickling_upstream, "127.0.0.1", 0)
|
||||
port = server.sockets[0].getsockname()[1]
|
||||
clock = FakeClock()
|
||||
closer = make_closer(clock)
|
||||
handler = AsyncHTTPHandler()
|
||||
handler = AsyncHTTPHandler(http2=http2)
|
||||
held_client = handler.client
|
||||
|
||||
closer.mark_owned(handler)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue