This commit is contained in:
Pruthvi Shetty 2026-10-27 02:25:45 +00:00 • committed by GitHub
commit 8e590546df
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 175 additions and 1 deletions

View file

@ -386,6 +386,9 @@ EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000
# Set to 0 for unlimited (not recommended for production)
AIOHTTP_CONNECTOR_LIMIT: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT", 1000)
AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500)
# Log a warning when a request waits at least this many seconds for a free aiohttp
# connection slot (pool saturation). Set to 0 or a negative value to disable.
AIOHTTP_POOL_WAIT_WARNING_SECONDS: Final = get_env_float("AIOHTTP_POOL_WAIT_WARNING_SECONDS", 1.0)
AIOHTTP_KEEPALIVE_TIMEOUT: Final = get_env_int("AIOHTTP_KEEPALIVE_TIMEOUT", 15)
AIOHTTP_TTL_DNS_CACHE: Final = get_env_int("AIOHTTP_TTL_DNS_CACHE", 300)
# TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs

View file

@ -0,0 +1,78 @@
"""
Diagnostics for aiohttp connection-pool saturation.
When the connector's ``limit`` / ``limit_per_host`` is reached, aiohttp queues
the request until a slot frees up. Without this, the only visible symptom is a
request timeout, which looks identical to a slow upstream. This module attaches
an aiohttp ``TraceConfig`` that logs a warning naming the host, the configured
limits, and how long the request waited for a connection slot.
"""
import time
from collections.abc import Awaitable, Callable
from types import SimpleNamespace
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_logger
from litellm.constants import (
AIOHTTP_CONNECTOR_LIMIT,
AIOHTTP_CONNECTOR_LIMIT_PER_HOST,
AIOHTTP_POOL_WAIT_WARNING_SECONDS,
)
if TYPE_CHECKING:
from aiohttp import ClientSession, TraceConfig
async def _on_request_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None:
url: Final = getattr(params, "url", None)
ctx.pool_wait_host = getattr(url, "host", None) or "unknown" # rebind-ok: aiohttp trace ctx holds per-request state
async def _on_connection_queued_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None:
ctx.pool_wait_started_at = time.monotonic() # rebind-ok: aiohttp trace ctx holds per-request state
def _make_on_connection_queued_end(
threshold_seconds: float,
) -> Callable[["ClientSession", SimpleNamespace, object], Awaitable[None]]:
async def _on_connection_queued_end(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None:
started_at: Final = getattr(ctx, "pool_wait_started_at", None)
if started_at is None:
return
waited: Final = time.monotonic() - started_at
if waited < threshold_seconds:
return
verbose_logger.warning(
"aiohttp connection pool saturated: request to host=%s waited %.2fs "
"for a free connection slot (limit=%s, limit_per_host=%s). The "
"upstream may be healthy; consider raising "
"AIOHTTP_CONNECTOR_LIMIT_PER_HOST / AIOHTTP_CONNECTOR_LIMIT or "
"investigating idle connections.",
getattr(ctx, "pool_wait_host", "unknown"),
waited,
AIOHTTP_CONNECTOR_LIMIT or "unlimited",
AIOHTTP_CONNECTOR_LIMIT_PER_HOST or "unlimited",
)
return _on_connection_queued_end
def build_pool_wait_trace_config(
threshold_seconds: float | None = None,
) -> "TraceConfig | None":
"""
Return a TraceConfig that warns when a request waits for a pooled
connection longer than ``threshold_seconds`` (defaults to
``AIOHTTP_POOL_WAIT_WARNING_SECONDS``). Returns None when disabled.
"""
effective_threshold: Final = AIOHTTP_POOL_WAIT_WARNING_SECONDS if threshold_seconds is None else threshold_seconds
if effective_threshold <= 0:
return None
from aiohttp import TraceConfig
trace_config: Final = TraceConfig()
trace_config.on_request_start.append(_on_request_start)
trace_config.on_connection_queued_start.append(_on_connection_queued_start)
trace_config.on_connection_queued_end.append(_make_on_connection_queued_end(effective_threshold))
return trace_config

View file

@ -1346,11 +1346,17 @@ class AsyncHTTPHandler:
if socket_factory is not None:
transport_connector_kwargs["socket_factory"] = socket_factory
from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import (
build_pool_wait_trace_config,
)
def session_factory() -> ClientSession:
pool_wait_trace_config: Final = build_pool_wait_trace_config()
return ClientSession(
connector=TCPConnector(**transport_connector_kwargs),
cookie_jar=DummyCookieJar(),
trust_env=trust_env,
trace_configs=[pool_wait_trace_config] if pool_wait_trace_config is not None else None,
)
# Use shared session if provided and valid

View file

@ -1398,7 +1398,16 @@ async def initialize_shared_aiohttp_session() -> "ClientSession | None":
connector_kwargs["socket_factory"] = socket_factory
connector: Final = TCPConnector(**connector_kwargs)
session: Final = ClientSession(connector=connector, cookie_jar=DummyCookieJar())
from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import (
build_pool_wait_trace_config,
)
pool_wait_trace_config: Final = build_pool_wait_trace_config()
session: Final = ClientSession(
connector=connector,
cookie_jar=DummyCookieJar(),
trace_configs=[pool_wait_trace_config] if pool_wait_trace_config is not None else None,
)
verbose_proxy_logger.info(
"SESSION REUSE: Created shared aiohttp session for connection pooling (ID: %s, limit=%s, limit_per_host=%s)",

View file

@ -0,0 +1,78 @@
"""Tests for the aiohttp pool-saturation warning (issue #46864)."""
import asyncio
import logging
import pytest
from aiohttp import ClientSession, ClientTimeout, TCPConnector, web
from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import (
build_pool_wait_trace_config,
)
async def _start_slow_server():
async def handler(request):
await asyncio.sleep(0.3)
return web.Response(text="ok")
app = web.Application()
app.router.add_get("/", handler)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
port = site._server.sockets[0].getsockname()[1]
return runner, f"http://127.0.0.1:{port}/"
def test_disabled_when_threshold_not_positive():
assert build_pool_wait_trace_config(threshold_seconds=0) is None
assert build_pool_wait_trace_config(threshold_seconds=-1) is None
assert build_pool_wait_trace_config(threshold_seconds=0.5) is not None
async def _run_concurrent(threshold, limit_per_host):
runner, url = await _start_slow_server()
try:
return await _fetch_twice(url, threshold, limit_per_host)
finally:
await runner.cleanup()
async def _fetch_twice(url, threshold, limit_per_host):
trace_config = build_pool_wait_trace_config(threshold_seconds=threshold)
async with ClientSession(
connector=TCPConnector(limit_per_host=limit_per_host),
trace_configs=[trace_config],
timeout=ClientTimeout(total=10),
) as session:
async def fetch():
async with session.get(url) as resp:
return await resp.text()
return await asyncio.gather(fetch(), fetch())
@pytest.mark.asyncio
async def test_warns_when_request_waits_for_pool_slot(caplog):
caplog.set_level(logging.WARNING, logger="LiteLLM")
results = await _run_concurrent(threshold=0.1, limit_per_host=1)
assert results == ["ok", "ok"]
messages = [r.getMessage() for r in caplog.records]
assert any("connection pool saturated" in m and "host=127.0.0.1" in m for m in messages), messages
@pytest.mark.asyncio
async def test_no_warning_when_pool_has_capacity(caplog):
caplog.set_level(logging.WARNING, logger="LiteLLM")
await _run_concurrent(threshold=0.1, limit_per_host=10)
assert not any("connection pool saturated" in r.getMessage() for r in caplog.records)
@pytest.mark.asyncio
async def test_no_warning_when_wait_below_threshold(caplog):
caplog.set_level(logging.WARNING, logger="LiteLLM")
await _run_concurrent(threshold=5.0, limit_per_host=1)
assert not any("connection pool saturated" in r.getMessage() for r in caplog.records)