mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 3413ac6a97 into e16d79d635
This commit is contained in:
commit
8e590546df
5 changed files with 175 additions and 1 deletions
|
|
@ -386,6 +386,9 @@ EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000
|
|||
# Set to 0 for unlimited (not recommended for production)
|
||||
AIOHTTP_CONNECTOR_LIMIT: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT", 1000)
|
||||
AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500)
|
||||
# Log a warning when a request waits at least this many seconds for a free aiohttp
|
||||
# connection slot (pool saturation). Set to 0 or a negative value to disable.
|
||||
AIOHTTP_POOL_WAIT_WARNING_SECONDS: Final = get_env_float("AIOHTTP_POOL_WAIT_WARNING_SECONDS", 1.0)
|
||||
AIOHTTP_KEEPALIVE_TIMEOUT: Final = get_env_int("AIOHTTP_KEEPALIVE_TIMEOUT", 15)
|
||||
AIOHTTP_TTL_DNS_CACHE: Final = get_env_int("AIOHTTP_TTL_DNS_CACHE", 300)
|
||||
# TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs
|
||||
|
|
|
|||
78
litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py
Normal file
78
litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""
|
||||
Diagnostics for aiohttp connection-pool saturation.
|
||||
|
||||
When the connector's ``limit`` / ``limit_per_host`` is reached, aiohttp queues
|
||||
the request until a slot frees up. Without this, the only visible symptom is a
|
||||
request timeout, which looks identical to a slow upstream. This module attaches
|
||||
an aiohttp ``TraceConfig`` that logs a warning naming the host, the configured
|
||||
limits, and how long the request waited for a connection slot.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
AIOHTTP_CONNECTOR_LIMIT,
|
||||
AIOHTTP_CONNECTOR_LIMIT_PER_HOST,
|
||||
AIOHTTP_POOL_WAIT_WARNING_SECONDS,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiohttp import ClientSession, TraceConfig
|
||||
|
||||
|
||||
async def _on_request_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None:
|
||||
url: Final = getattr(params, "url", None)
|
||||
ctx.pool_wait_host = getattr(url, "host", None) or "unknown" # rebind-ok: aiohttp trace ctx holds per-request state
|
||||
|
||||
|
||||
async def _on_connection_queued_start(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None:
|
||||
ctx.pool_wait_started_at = time.monotonic() # rebind-ok: aiohttp trace ctx holds per-request state
|
||||
|
||||
|
||||
def _make_on_connection_queued_end(
|
||||
threshold_seconds: float,
|
||||
) -> Callable[["ClientSession", SimpleNamespace, object], Awaitable[None]]:
|
||||
async def _on_connection_queued_end(session: "ClientSession", ctx: SimpleNamespace, params: object) -> None:
|
||||
started_at: Final = getattr(ctx, "pool_wait_started_at", None)
|
||||
if started_at is None:
|
||||
return
|
||||
waited: Final = time.monotonic() - started_at
|
||||
if waited < threshold_seconds:
|
||||
return
|
||||
verbose_logger.warning(
|
||||
"aiohttp connection pool saturated: request to host=%s waited %.2fs "
|
||||
"for a free connection slot (limit=%s, limit_per_host=%s). The "
|
||||
"upstream may be healthy; consider raising "
|
||||
"AIOHTTP_CONNECTOR_LIMIT_PER_HOST / AIOHTTP_CONNECTOR_LIMIT or "
|
||||
"investigating idle connections.",
|
||||
getattr(ctx, "pool_wait_host", "unknown"),
|
||||
waited,
|
||||
AIOHTTP_CONNECTOR_LIMIT or "unlimited",
|
||||
AIOHTTP_CONNECTOR_LIMIT_PER_HOST or "unlimited",
|
||||
)
|
||||
|
||||
return _on_connection_queued_end
|
||||
|
||||
|
||||
def build_pool_wait_trace_config(
|
||||
threshold_seconds: float | None = None,
|
||||
) -> "TraceConfig | None":
|
||||
"""
|
||||
Return a TraceConfig that warns when a request waits for a pooled
|
||||
connection longer than ``threshold_seconds`` (defaults to
|
||||
``AIOHTTP_POOL_WAIT_WARNING_SECONDS``). Returns None when disabled.
|
||||
"""
|
||||
effective_threshold: Final = AIOHTTP_POOL_WAIT_WARNING_SECONDS if threshold_seconds is None else threshold_seconds
|
||||
if effective_threshold <= 0:
|
||||
return None
|
||||
from aiohttp import TraceConfig
|
||||
|
||||
trace_config: Final = TraceConfig()
|
||||
trace_config.on_request_start.append(_on_request_start)
|
||||
trace_config.on_connection_queued_start.append(_on_connection_queued_start)
|
||||
trace_config.on_connection_queued_end.append(_make_on_connection_queued_end(effective_threshold))
|
||||
return trace_config
|
||||
|
|
@ -1346,11 +1346,17 @@ class AsyncHTTPHandler:
|
|||
if socket_factory is not None:
|
||||
transport_connector_kwargs["socket_factory"] = socket_factory
|
||||
|
||||
from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import (
|
||||
build_pool_wait_trace_config,
|
||||
)
|
||||
|
||||
def session_factory() -> ClientSession:
|
||||
pool_wait_trace_config: Final = build_pool_wait_trace_config()
|
||||
return ClientSession(
|
||||
connector=TCPConnector(**transport_connector_kwargs),
|
||||
cookie_jar=DummyCookieJar(),
|
||||
trust_env=trust_env,
|
||||
trace_configs=[pool_wait_trace_config] if pool_wait_trace_config is not None else None,
|
||||
)
|
||||
|
||||
# Use shared session if provided and valid
|
||||
|
|
|
|||
|
|
@ -1398,7 +1398,16 @@ async def initialize_shared_aiohttp_session() -> "ClientSession | None":
|
|||
connector_kwargs["socket_factory"] = socket_factory
|
||||
|
||||
connector: Final = TCPConnector(**connector_kwargs)
|
||||
session: Final = ClientSession(connector=connector, cookie_jar=DummyCookieJar())
|
||||
from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import (
|
||||
build_pool_wait_trace_config,
|
||||
)
|
||||
|
||||
pool_wait_trace_config: Final = build_pool_wait_trace_config()
|
||||
session: Final = ClientSession(
|
||||
connector=connector,
|
||||
cookie_jar=DummyCookieJar(),
|
||||
trace_configs=[pool_wait_trace_config] if pool_wait_trace_config is not None else None,
|
||||
)
|
||||
|
||||
verbose_proxy_logger.info(
|
||||
"SESSION REUSE: Created shared aiohttp session for connection pooling (ID: %s, limit=%s, limit_per_host=%s)",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,78 @@
|
|||
"""Tests for the aiohttp pool-saturation warning (issue #46864)."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
from aiohttp import ClientSession, ClientTimeout, TCPConnector, web
|
||||
|
||||
from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import (
|
||||
build_pool_wait_trace_config,
|
||||
)
|
||||
|
||||
|
||||
async def _start_slow_server():
|
||||
async def handler(request):
|
||||
await asyncio.sleep(0.3)
|
||||
return web.Response(text="ok")
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_get("/", handler)
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, "127.0.0.1", 0)
|
||||
await site.start()
|
||||
port = site._server.sockets[0].getsockname()[1]
|
||||
return runner, f"http://127.0.0.1:{port}/"
|
||||
|
||||
|
||||
def test_disabled_when_threshold_not_positive():
|
||||
assert build_pool_wait_trace_config(threshold_seconds=0) is None
|
||||
assert build_pool_wait_trace_config(threshold_seconds=-1) is None
|
||||
assert build_pool_wait_trace_config(threshold_seconds=0.5) is not None
|
||||
|
||||
|
||||
async def _run_concurrent(threshold, limit_per_host):
|
||||
runner, url = await _start_slow_server()
|
||||
try:
|
||||
return await _fetch_twice(url, threshold, limit_per_host)
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
|
||||
async def _fetch_twice(url, threshold, limit_per_host):
|
||||
trace_config = build_pool_wait_trace_config(threshold_seconds=threshold)
|
||||
async with ClientSession(
|
||||
connector=TCPConnector(limit_per_host=limit_per_host),
|
||||
trace_configs=[trace_config],
|
||||
timeout=ClientTimeout(total=10),
|
||||
) as session:
|
||||
|
||||
async def fetch():
|
||||
async with session.get(url) as resp:
|
||||
return await resp.text()
|
||||
|
||||
return await asyncio.gather(fetch(), fetch())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_warns_when_request_waits_for_pool_slot(caplog):
|
||||
caplog.set_level(logging.WARNING, logger="LiteLLM")
|
||||
results = await _run_concurrent(threshold=0.1, limit_per_host=1)
|
||||
assert results == ["ok", "ok"]
|
||||
messages = [r.getMessage() for r in caplog.records]
|
||||
assert any("connection pool saturated" in m and "host=127.0.0.1" in m for m in messages), messages
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_warning_when_pool_has_capacity(caplog):
|
||||
caplog.set_level(logging.WARNING, logger="LiteLLM")
|
||||
await _run_concurrent(threshold=0.1, limit_per_host=10)
|
||||
assert not any("connection pool saturated" in r.getMessage() for r in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_warning_when_wait_below_threshold(caplog):
|
||||
caplog.set_level(logging.WARNING, logger="LiteLLM")
|
||||
await _run_concurrent(threshold=5.0, limit_per_host=1)
|
||||
assert not any("connection pool saturated" in r.getMessage() for r in caplog.records)
|
||||
Loading…
Add table
Reference in a new issue