From 6e390801b3be9423ca918ed5938758a025075323 Mon Sep 17 00:00:00 2001 From: Pruthvi Shetty <13150396+pruthvishetty@users.noreply.github.com> Date: Fri, 16 Oct 2026 02:56:28 -0700 Subject: [PATCH 1/3] feat(http): warn when aiohttp connection pool is saturated --- litellm/constants.py | 3 + .../custom_httpx/aiohttp_pool_diagnostics.py | 76 ++++++++++++++++++ litellm/llms/custom_httpx/http_handler.py | 6 ++ litellm/proxy/proxy_server.py | 11 ++- .../test_aiohttp_pool_diagnostics.py | 78 +++++++++++++++++++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py create mode 100644 tests/unit/llms/custom_httpx/test_aiohttp_pool_diagnostics.py diff --git a/litellm/constants.py b/litellm/constants.py index 665ab771726..d1cd6f89366 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -352,6 +352,9 @@ EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000 # Set to 0 for unlimited (not recommended for production) AIOHTTP_CONNECTOR_LIMIT: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT", 1000) AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500) +# Log a warning when a request waits at least this many seconds for a free aiohttp +# connection slot (pool saturation). Set to 0 or a negative value to disable. +AIOHTTP_POOL_WAIT_WARNING_SECONDS: Final = float(os.getenv("AIOHTTP_POOL_WAIT_WARNING_SECONDS", "1.0")) AIOHTTP_KEEPALIVE_TIMEOUT: Final = get_env_int("AIOHTTP_KEEPALIVE_TIMEOUT", 15) AIOHTTP_TTL_DNS_CACHE: Final = get_env_int("AIOHTTP_TTL_DNS_CACHE", 300) # TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs diff --git a/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py b/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py new file mode 100644 index 00000000000..57741ef4e1f --- /dev/null +++ b/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py @@ -0,0 +1,76 @@ +""" +Diagnostics for aiohttp connection-pool saturation. + +When the connector's ``limit`` / ``limit_per_host`` is reached, aiohttp queues +the request until a slot frees up. Without this, the only visible symptom is a +request timeout, which looks identical to a slow upstream. This module attaches +an aiohttp ``TraceConfig`` that logs a warning naming the host, the configured +limits, and how long the request waited for a connection slot. +""" + +import time +from types import SimpleNamespace +from typing import TYPE_CHECKING + +from litellm._logging import verbose_logger +from litellm.constants import ( + AIOHTTP_CONNECTOR_LIMIT, + AIOHTTP_CONNECTOR_LIMIT_PER_HOST, + AIOHTTP_POOL_WAIT_WARNING_SECONDS, +) + +if TYPE_CHECKING: + from aiohttp import ClientSession, TraceConfig + + +async def _on_request_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: + url = getattr(params, "url", None) + ctx.pool_wait_host = getattr(url, "host", None) or "unknown" + + +async def _on_connection_queued_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: + ctx.pool_wait_started_at = time.monotonic() + + +def _make_on_connection_queued_end(threshold_seconds: float): + async def _on_connection_queued_end(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: + started_at = getattr(ctx, "pool_wait_started_at", None) + if started_at is None: + return + waited = time.monotonic() - started_at + if waited < threshold_seconds: + return + verbose_logger.warning( + "aiohttp connection pool saturated: request to host=%s waited %.2fs " + "for a free connection slot (limit=%s, limit_per_host=%s). The " + "upstream may be healthy; consider raising " + "AIOHTTP_CONNECTOR_LIMIT_PER_HOST / AIOHTTP_CONNECTOR_LIMIT or " + "investigating idle connections.", + getattr(ctx, "pool_wait_host", "unknown"), + waited, + AIOHTTP_CONNECTOR_LIMIT or "unlimited", + AIOHTTP_CONNECTOR_LIMIT_PER_HOST or "unlimited", + ) + + return _on_connection_queued_end + + +def build_pool_wait_trace_config( + threshold_seconds: float | None = None, +) -> "TraceConfig | None": + """ + Return a TraceConfig that warns when a request waits for a pooled + connection longer than ``threshold_seconds`` (defaults to + ``AIOHTTP_POOL_WAIT_WARNING_SECONDS``). Returns None when disabled. + """ + if threshold_seconds is None: + threshold_seconds = AIOHTTP_POOL_WAIT_WARNING_SECONDS + if threshold_seconds <= 0: + return None + from aiohttp import TraceConfig + + trace_config = TraceConfig() + trace_config.on_request_start.append(_on_request_start) + trace_config.on_connection_queued_start.append(_on_connection_queued_start) + trace_config.on_connection_queued_end.append(_make_on_connection_queued_end(threshold_seconds)) + return trace_config diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index cffeb44f12c..11ab6799594 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1350,11 +1350,17 @@ class AsyncHTTPHandler: if socket_factory is not None: transport_connector_kwargs["socket_factory"] = socket_factory + from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import ( + build_pool_wait_trace_config, + ) + def session_factory() -> ClientSession: + pool_wait_trace_config = build_pool_wait_trace_config() return ClientSession( connector=TCPConnector(**transport_connector_kwargs), cookie_jar=DummyCookieJar(), trust_env=trust_env, + trace_configs=[pool_wait_trace_config] if pool_wait_trace_config is not None else None, ) # Use shared session if provided and valid diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bbd9d96b84b..e610d730b0c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1329,7 +1329,16 @@ async def initialize_shared_aiohttp_session() -> "ClientSession | None": connector_kwargs["socket_factory"] = socket_factory connector: Final = TCPConnector(**connector_kwargs) - session: Final = ClientSession(connector=connector, cookie_jar=DummyCookieJar()) + from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import ( + build_pool_wait_trace_config, + ) + + pool_wait_trace_config: Final = build_pool_wait_trace_config() + session: Final = ClientSession( + connector=connector, + cookie_jar=DummyCookieJar(), + trace_configs=[pool_wait_trace_config] if pool_wait_trace_config is not None else None, + ) verbose_proxy_logger.info( "SESSION REUSE: Created shared aiohttp session for connection pooling (ID: %s, limit=%s, limit_per_host=%s)", diff --git a/tests/unit/llms/custom_httpx/test_aiohttp_pool_diagnostics.py b/tests/unit/llms/custom_httpx/test_aiohttp_pool_diagnostics.py new file mode 100644 index 00000000000..5c13121e7ec --- /dev/null +++ b/tests/unit/llms/custom_httpx/test_aiohttp_pool_diagnostics.py @@ -0,0 +1,78 @@ +"""Tests for the aiohttp pool-saturation warning (issue #46864).""" + +import asyncio +import logging + +import pytest +from aiohttp import ClientSession, ClientTimeout, TCPConnector, web + +from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import ( + build_pool_wait_trace_config, +) + + +async def _start_slow_server(): + async def handler(request): + await asyncio.sleep(0.3) + return web.Response(text="ok") + + app = web.Application() + app.router.add_get("/", handler) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + port = site._server.sockets[0].getsockname()[1] + return runner, f"http://127.0.0.1:{port}/" + + +def test_disabled_when_threshold_not_positive(): + assert build_pool_wait_trace_config(threshold_seconds=0) is None + assert build_pool_wait_trace_config(threshold_seconds=-1) is None + assert build_pool_wait_trace_config(threshold_seconds=0.5) is not None + + +async def _run_concurrent(threshold, limit_per_host): + runner, url = await _start_slow_server() + try: + return await _fetch_twice(url, threshold, limit_per_host) + finally: + await runner.cleanup() + + +async def _fetch_twice(url, threshold, limit_per_host): + trace_config = build_pool_wait_trace_config(threshold_seconds=threshold) + async with ClientSession( + connector=TCPConnector(limit_per_host=limit_per_host), + trace_configs=[trace_config], + timeout=ClientTimeout(total=10), + ) as session: + + async def fetch(): + async with session.get(url) as resp: + return await resp.text() + + return await asyncio.gather(fetch(), fetch()) + + +@pytest.mark.asyncio +async def test_warns_when_request_waits_for_pool_slot(caplog): + caplog.set_level(logging.WARNING, logger="LiteLLM") + results = await _run_concurrent(threshold=0.1, limit_per_host=1) + assert results == ["ok", "ok"] + messages = [r.getMessage() for r in caplog.records] + assert any("connection pool saturated" in m and "host=127.0.0.1" in m for m in messages), messages + + +@pytest.mark.asyncio +async def test_no_warning_when_pool_has_capacity(caplog): + caplog.set_level(logging.WARNING, logger="LiteLLM") + await _run_concurrent(threshold=0.1, limit_per_host=10) + assert not any("connection pool saturated" in r.getMessage() for r in caplog.records) + + +@pytest.mark.asyncio +async def test_no_warning_when_wait_below_threshold(caplog): + caplog.set_level(logging.WARNING, logger="LiteLLM") + await _run_concurrent(threshold=5.0, limit_per_host=1) + assert not any("connection pool saturated" in r.getMessage() for r in caplog.records) From fe624a861228571df0c49f6f73ee3c7353ff0e48 Mon Sep 17 00:00:00 2001 From: Pruthvi Shetty <13150396+pruthvishetty@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:11:44 -0700 Subject: [PATCH 2/3] fix(http): harden aiohttp pool wait warning config Use get_env_float so blank env values do not crash import, annotate the queued-end factory for ruff ANN202, and keep the warning threshold default. --- litellm/constants.py | 2 +- litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index d1cd6f89366..fa3de18d6cc 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -354,7 +354,7 @@ AIOHTTP_CONNECTOR_LIMIT: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT", 1000) AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500) # Log a warning when a request waits at least this many seconds for a free aiohttp # connection slot (pool saturation). Set to 0 or a negative value to disable. -AIOHTTP_POOL_WAIT_WARNING_SECONDS: Final = float(os.getenv("AIOHTTP_POOL_WAIT_WARNING_SECONDS", "1.0")) +AIOHTTP_POOL_WAIT_WARNING_SECONDS: Final = get_env_float("AIOHTTP_POOL_WAIT_WARNING_SECONDS", 1.0) AIOHTTP_KEEPALIVE_TIMEOUT: Final = get_env_int("AIOHTTP_KEEPALIVE_TIMEOUT", 15) AIOHTTP_TTL_DNS_CACHE: Final = get_env_int("AIOHTTP_TTL_DNS_CACHE", 300) # TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs diff --git a/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py b/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py index 57741ef4e1f..6f304e6da04 100644 --- a/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py +++ b/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py @@ -9,6 +9,7 @@ limits, and how long the request waited for a connection slot. """ import time +from collections.abc import Awaitable, Callable from types import SimpleNamespace from typing import TYPE_CHECKING @@ -32,7 +33,9 @@ async def _on_connection_queued_start(session: "ClientSession", ctx: SimpleNames ctx.pool_wait_started_at = time.monotonic() -def _make_on_connection_queued_end(threshold_seconds: float): +def _make_on_connection_queued_end( + threshold_seconds: float, +) -> Callable[["ClientSession", SimpleNamespace, object], Awaitable[None]]: async def _on_connection_queued_end(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: started_at = getattr(ctx, "pool_wait_started_at", None) if started_at is None: From 3413ac6a9783206e0dc5c093a885d94824c006cc Mon Sep 17 00:00:00 2001 From: Pruthvi Shetty <13150396+pruthvishetty@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:45:10 -0700 Subject: [PATCH 3/3] fix(http): satisfy type-discipline gate (Final + rebind-ok on trace ctx) --- .../custom_httpx/aiohttp_pool_diagnostics.py | 21 +++++++++---------- litellm/llms/custom_httpx/http_handler.py | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py b/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py index 6f304e6da04..4c42378d464 100644 --- a/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py +++ b/litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py @@ -11,7 +11,7 @@ limits, and how long the request waited for a connection slot. import time from collections.abc import Awaitable, Callable from types import SimpleNamespace -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final from litellm._logging import verbose_logger from litellm.constants import ( @@ -25,22 +25,22 @@ if TYPE_CHECKING: async def _on_request_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: - url = getattr(params, "url", None) - ctx.pool_wait_host = getattr(url, "host", None) or "unknown" + url: Final = getattr(params, "url", None) + ctx.pool_wait_host = getattr(url, "host", None) or "unknown" # rebind-ok: aiohttp trace ctx holds per-request state async def _on_connection_queued_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: - ctx.pool_wait_started_at = time.monotonic() + ctx.pool_wait_started_at = time.monotonic() # rebind-ok: aiohttp trace ctx holds per-request state def _make_on_connection_queued_end( threshold_seconds: float, ) -> Callable[["ClientSession", SimpleNamespace, object], Awaitable[None]]: async def _on_connection_queued_end(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None: - started_at = getattr(ctx, "pool_wait_started_at", None) + started_at: Final = getattr(ctx, "pool_wait_started_at", None) if started_at is None: return - waited = time.monotonic() - started_at + waited: Final = time.monotonic() - started_at if waited < threshold_seconds: return verbose_logger.warning( @@ -66,14 +66,13 @@ def build_pool_wait_trace_config( connection longer than ``threshold_seconds`` (defaults to ``AIOHTTP_POOL_WAIT_WARNING_SECONDS``). Returns None when disabled. """ - if threshold_seconds is None: - threshold_seconds = AIOHTTP_POOL_WAIT_WARNING_SECONDS - if threshold_seconds <= 0: + effective_threshold: Final = AIOHTTP_POOL_WAIT_WARNING_SECONDS if threshold_seconds is None else threshold_seconds + if effective_threshold <= 0: return None from aiohttp import TraceConfig - trace_config = TraceConfig() + trace_config: Final = TraceConfig() trace_config.on_request_start.append(_on_request_start) trace_config.on_connection_queued_start.append(_on_connection_queued_start) - trace_config.on_connection_queued_end.append(_make_on_connection_queued_end(threshold_seconds)) + trace_config.on_connection_queued_end.append(_make_on_connection_queued_end(effective_threshold)) return trace_config diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 11ab6799594..5fb19eb8f7b 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -1355,7 +1355,7 @@ class AsyncHTTPHandler: ) def session_factory() -> ClientSession: - pool_wait_trace_config = build_pool_wait_trace_config() + pool_wait_trace_config: Final = build_pool_wait_trace_config() return ClientSession( connector=TCPConnector(**transport_connector_kwargs), cookie_jar=DummyCookieJar(),