diff --git a/litellm/constants.py b/litellm/constants.py index 2901b7d8948..b79baf397d1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -386,6 +386,9 @@ EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000 # Set to 0 for unlimited (not recommended for production) AIOHTTP_CONNECTOR_LIMIT: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT", 1000) AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500) +# Log a warning when a request waits at least this many seconds for a free aiohttp +# connection slot (pool saturation). Set to 0 or a negative value to disable. +AIOHTTP_POOL_WAIT_WARNING_SECONDS: Final = get_env_float("AIOHTTP_POOL_WAIT_WARNING_SECONDS", 1.0) AIOHTTP_KEEPALIVE_TIMEOUT: Final = get_env_int("AIOHTTP_KEEPALIVE_TIMEOUT", 15) AIOHTTP_TTL_DNS_CACHE: Final = get_env_int("AIOHTTP_TTL_DNS_CACHE", 300) # TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs diff --git a/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py b/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py new file mode 100644 index 00000000000..4c42378d464 --- /dev/null +++ b/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py @@ -0,0 +1,78 @@ +""" +Diagnostics for aiohttp connection-pool saturation. + +When the connector's ``limit`` / ``limit_per_host`` is reached, aiohttp queues +the request until a slot frees up. Without this, the only visible symptom is a +request timeout, which looks identical to a slow upstream. This module attaches +an aiohttp ``TraceConfig`` that logs a warning naming the host, the configured +limits, and how long the request waited for a connection slot. +""" + +import time +from collections.abc import Awaitable, Callable +from types import SimpleNamespace +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_logger +from litellm.constants import ( + AIOHTTP_CONNECTOR_LIMIT, + AIOHTTP_CONNECTOR_LIMIT_PER_HOST, + AIOHTTP_POOL_WAIT_WARNING_SECONDS, +) + +if TYPE_CHECKING: + from aiohttp import ClientSession, TraceConfig + + +async def _on_request_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: + url: Final = getattr(params, "url", None) + ctx.pool_wait_host = getattr(url, "host", None) or "unknown" # rebind-ok: aiohttp trace ctx holds per-request state + + +async def _on_connection_queued_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: + ctx.pool_wait_started_at = time.monotonic() # rebind-ok: aiohttp trace ctx holds per-request state + + +def _make_on_connection_queued_end( + threshold_seconds: float, +) -> Callable[["ClientSession", SimpleNamespace, object], Awaitable[None]]: + async def _on_connection_queued_end(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: + started_at: Final = getattr(ctx, "pool_wait_started_at", None) + if started_at is None: + return + waited: Final = time.monotonic() - started_at + if waited < threshold_seconds: + return + verbose_logger.warning( + "aiohttp connection pool saturated: request to host=%s waited %.2fs " + "for a free connection slot (limit=%s, limit_per_host=%s). The " + "upstream may be healthy; consider raising " + "AIOHTTP_CONNECTOR_LIMIT_PER_HOST / AIOHTTP_CONNECTOR_LIMIT or " + "investigating idle connections.", + getattr(ctx, "pool_wait_host", "unknown"), + waited, + AIOHTTP_CONNECTOR_LIMIT or "unlimited", + AIOHTTP_CONNECTOR_LIMIT_PER_HOST or "unlimited", + ) + + return _on_connection_queued_end + + +def build_pool_wait_trace_config( + threshold_seconds: float | None = None, +) -> "TraceConfig | None": + """ + Return a TraceConfig that warns when a request waits for a pooled + connection longer than ``threshold_seconds`` (defaults to + ``AIOHTTP_POOL_WAIT_WARNING_SECONDS``). Returns None when disabled. + """ + effective_threshold: Final = AIOHTTP_POOL_WAIT_WARNING_SECONDS if threshold_seconds is None else threshold_seconds + if effective_threshold <= 0: + return None + from aiohttp import TraceConfig + + trace_config: Final = TraceConfig() + trace_config.on_request_start.append(_on_request_start) + trace_config.on_connection_queued_start.append(_on_connection_queued_start) + trace_config.on_connection_queued_end.append(_make_on_connection_queued_end(effective_threshold)) + return trace_config diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0b64885df45..ec51b29054c 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1346,11 +1346,17 @@ class AsyncHTTPHandler: if socket_factory is not None: transport_connector_kwargs["socket_factory"] = socket_factory + from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import ( + build_pool_wait_trace_config, + ) + def session_factory() -> ClientSession: + pool_wait_trace_config: Final = build_pool_wait_trace_config() return ClientSession( connector=TCPConnector(**transport_connector_kwargs), cookie_jar=DummyCookieJar(), trust_env=trust_env, + trace_configs=[pool_wait_trace_config] if pool_wait_trace_config is not None else None, ) # Use shared session if provided and valid diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ca0f8892b9d..ea12fd879c3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1398,7 +1398,16 @@ async def initialize_shared_aiohttp_session() -> "ClientSession | None": connector_kwargs["socket_factory"] = socket_factory connector: Final = TCPConnector(**connector_kwargs) - session: Final = ClientSession(connector=connector, cookie_jar=DummyCookieJar()) + from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import ( + build_pool_wait_trace_config, + ) + + pool_wait_trace_config: Final = build_pool_wait_trace_config() + session: Final = ClientSession( + connector=connector, + cookie_jar=DummyCookieJar(), + trace_configs=[pool_wait_trace_config] if pool_wait_trace_config is not None else None, + ) verbose_proxy_logger.info( "SESSION REUSE: Created shared aiohttp session for connection pooling (ID: %s, limit=%s, limit_per_host=%s)", diff --git a/tests/unit/llms/custom_httpx/test_aiohttp_pool_diagnostics.py b/tests/unit/llms/custom_httpx/test_aiohttp_pool_diagnostics.py new file mode 100644 index 00000000000..5c13121e7ec --- /dev/null +++ b/tests/unit/llms/custom_httpx/test_aiohttp_pool_diagnostics.py @@ -0,0 +1,78 @@ +"""Tests for the aiohttp pool-saturation warning (issue #46864).""" + +import asyncio +import logging + +import pytest +from aiohttp import ClientSession, ClientTimeout, TCPConnector, web + +from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import ( + build_pool_wait_trace_config, +) + + +async def _start_slow_server(): + async def handler(request): + await asyncio.sleep(0.3) + return web.Response(text="ok") + + app = web.Application() + app.router.add_get("/", handler) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] + return runner, f"http://127.0.0.1:{port}/" + + +def test_disabled_when_threshold_not_positive(): + assert build_pool_wait_trace_config(threshold_seconds=0) is None + assert build_pool_wait_trace_config(threshold_seconds=-1) is None + assert build_pool_wait_trace_config(threshold_seconds=0.5) is not None + + +async def _run_concurrent(threshold, limit_per_host): + runner, url = await _start_slow_server() + try: + return await _fetch_twice(url, threshold, limit_per_host) + finally: + await runner.cleanup() + + +async def _fetch_twice(url, threshold, limit_per_host): + trace_config = build_pool_wait_trace_config(threshold_seconds=threshold) + async with ClientSession( + connector=TCPConnector(limit_per_host=limit_per_host), + trace_configs=[trace_config], + timeout=ClientTimeout(total=10), + ) as session: + + async def fetch(): + async with session.get(url) as resp: + return await resp.text() + + return await asyncio.gather(fetch(), fetch()) + + +@pytest.mark.asyncio +async def test_warns_when_request_waits_for_pool_slot(caplog): + caplog.set_level(logging.WARNING, logger="LiteLLM") + results = await _run_concurrent(threshold=0.1, limit_per_host=1) + assert results == ["ok", "ok"] + messages = [r.getMessage() for r in caplog.records] + assert any("connection pool saturated" in m and "host=127.0.0.1" in m for m in messages), messages + + +@pytest.mark.asyncio +async def test_no_warning_when_pool_has_capacity(caplog): + caplog.set_level(logging.WARNING, logger="LiteLLM") + await _run_concurrent(threshold=0.1, limit_per_host=10) + assert not any("connection pool saturated" in r.getMessage() for r in caplog.records) + + +@pytest.mark.asyncio +async def test_no_warning_when_wait_below_threshold(caplog): + caplog.set_level(logging.WARNING, logger="LiteLLM") + await _run_concurrent(threshold=5.0, limit_per_host=1) + assert not any("connection pool saturated" in r.getMessage() for r in caplog.records)