diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 3bfeba2e394..b8621b10ef9 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -408,14 +408,17 @@ def _lazy_import_http_handlers(name: str) -> Any: if name == "module_level_aclient": # Create an async HTTP client using the factory function - from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.llms.custom_httpx.http_handler import ( + MODULE_LEVEL_ASYNC_HTTP_CLIENT_PROVIDER_ID, + get_async_httpx_client, + ) # Get timeout from module config (if set) timeout = _globals.get("request_timeout") params = {"timeout": timeout, "client_alias": "module level aclient"} # Create the client instance - provider_id = cast(Any, "litellm_module_level_client") + provider_id = cast(Any, MODULE_LEVEL_ASYNC_HTTP_CLIENT_PROVIDER_ID) async_client = get_async_httpx_client( llm_provider=provider_id, params=params, diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 3dfef07d426..a9abb7427fe 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -69,6 +69,7 @@ headers = get_default_headers() # https://www.python-httpx.org/advanced/timeouts _DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0) +MODULE_LEVEL_ASYNC_HTTP_CLIENT_PROVIDER_ID = "litellm_module_level_client" def _prepare_request_data_and_content( @@ -1210,6 +1211,21 @@ def get_async_httpx_client( Caches the new client and returns it. """ + if llm_provider == MODULE_LEVEL_ASYNC_HTTP_CLIENT_PROVIDER_ID: + # module_level_aclient is a process-wide singleton and should not be + # evicted from bounded in-memory cache under high traffic. + if params is not None: + handler_params = { + k: v for k, v in params.items() if k != "disable_aiohttp_transport" + } + handler_params["shared_session"] = shared_session + return AsyncHTTPHandler(**handler_params) + + return AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0), + shared_session=shared_session, + ) + _params_key_name = "" if params is not None: for key, value in params.items(): diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index b8922846e82..a0efe8d0c4b 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -10,8 +10,13 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest import redis.asyncio as async_redis +import litellm from litellm._redis import get_redis_async_client, get_redis_connection_pool from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.llms.custom_httpx.http_handler import ( + MODULE_LEVEL_ASYNC_HTTP_CLIENT_PROVIDER_ID, + get_async_httpx_client, +) def test_url_config_uses_passed_pool(): @@ -165,3 +170,48 @@ async def test_eviction_non_closeable_safe(): # If we got here without exception, the test passes assert cache.get_cache(key="int-val") == 42 + + +@pytest.mark.asyncio +async def test_module_level_async_client_not_stored_in_bounded_cache(): + """ + module_level_aclient is a process-wide singleton and should not be tied to + the bounded in-memory cache used for per-provider ephemeral clients. + """ + module_client = litellm.module_level_aclient + try: + cache_keys = list(getattr(litellm.in_memory_llm_clients_cache, "cache_dict", {}).keys()) + assert all( + MODULE_LEVEL_ASYNC_HTTP_CLIENT_PROVIDER_ID not in key for key in cache_keys + ) + assert module_client.client.is_closed is False + finally: + await module_client.close() + litellm.__dict__.pop("module_level_aclient", None) + + +@pytest.mark.asyncio +async def test_module_level_async_client_survives_cache_eviction_pressure(): + """ + Repro for RuntimeError("client has been closed") seen under traffic: + evicting many cached clients must not close module_level_aclient. + """ + module_client = litellm.module_level_aclient + cache = litellm.in_memory_llm_clients_cache + original_max_size = cache.max_size_in_memory + original_default_ttl = cache.default_ttl + + try: + cache.max_size_in_memory = 2 + cache.default_ttl = 3600 + + get_async_httpx_client("eviction_provider_1", params={"timeout": None}) + get_async_httpx_client("eviction_provider_2", params={"timeout": None}) + await asyncio.sleep(0.05) + + assert module_client.client.is_closed is False + finally: + cache.max_size_in_memory = original_max_size + cache.default_ttl = original_default_ttl + await module_client.close() + litellm.__dict__.pop("module_level_aclient", None)