diff --git a/litellm/__init__.py b/litellm/__init__.py index 05968e8a7c0..bd7b03064dd 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -26,7 +26,6 @@ from typing import ( ) from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams from litellm.types.integrations.datadog import DatadogInitParams -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.caching.caching import Cache, DualCache, RedisCache, InMemoryCache from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES @@ -422,10 +421,6 @@ disable_aiohttp_trust_env: bool = ( force_ipv4: bool = ( False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6. ) -module_level_aclient = AsyncHTTPHandler( - timeout=request_timeout, client_alias="module level aclient" -) -module_level_client = HTTPHandler(timeout=request_timeout) #### RETRIES #### num_retries: Optional[int] = None # per model endpoint @@ -1520,6 +1515,7 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: if TYPE_CHECKING: from litellm.types.utils import ModelInfo as _ModelInfoType + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] @@ -1560,13 +1556,18 @@ if TYPE_CHECKING: # Response types - truly lazy loaded only (not in main.py or elsewhere) ModelResponseListIterator: Type[Any] + # HTTP handler singletons (created lazily via __getattr__ at runtime) + module_level_aclient: AsyncHTTPHandler + module_level_client: HTTPHandler + def __getattr__(name: str) -> Any: - """Lazy import handler for cost_calculator and litellm_logging functions.""" + """Lazy import handler""" from ._lazy_imports import ( COST_CALCULATOR_NAMES, LITELLM_LOGGING_NAMES, UTILS_NAMES, + HTTP_HANDLER_NAMES, ) # Lazy load cost_calculator functions @@ -1584,6 +1585,12 @@ def __getattr__(name: str) -> Any: from ._lazy_imports import _lazy_import_utils return _lazy_import_utils(name) + # Lazy-load HTTP handler singletons used across the codebase + if name in HTTP_HANDLER_NAMES: + from ._lazy_imports import _lazy_import_http_handlers + + return _lazy_import_http_handlers(name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index b87b9c955fb..17772682599 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, cast import sys def _get_litellm_globals() -> dict: @@ -34,6 +34,12 @@ UTILS_NAMES = ( "ModelResponseListIterator", "get_valid_models", ) +# HTTP handler names that support lazy loading via _lazy_import_http_handlers +HTTP_HANDLER_NAMES = ( + "module_level_aclient", + "module_level_client", +) + # Lazy import for utils module - imports only the requested item by name. # Note: PLR0915 (too many statements) is suppressed because the many if statements # are intentional - each attribute is imported individually only when requested, @@ -278,4 +284,36 @@ def _lazy_import_litellm_logging(name: str) -> Any: _globals["modify_integration"] = _modify_integration return _modify_integration - raise AttributeError(f"Litellm logging lazy import: unknown attribute {name!r}") \ No newline at end of file + raise AttributeError(f"Litellm logging lazy import: unknown attribute {name!r}") + + +def _lazy_import_http_handlers(name: str) -> Any: + """Lazy import and instantiate module-level HTTP handlers.""" + _globals = _get_litellm_globals() + + if name == "module_level_aclient": + # Use shared async client factory instead of directly instantiating AsyncHTTPHandler + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + + timeout = _globals.get("request_timeout") + params = {"timeout": timeout, "client_alias": "module level aclient"} + # llm_provider is only used for cache keying; use a string identifier but + # cast to Any so static type checkers don't complain about the literal. + provider_id = cast(Any, "litellm_module_level_client") + async_client = get_async_httpx_client( + llm_provider=provider_id, + params=params, + ) + _globals["module_level_aclient"] = async_client + return async_client + + if name == "module_level_client": + # Import handler type locally to avoid heavy imports at module load time + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + timeout = _globals.get("request_timeout") + sync_client = HTTPHandler(timeout=timeout) + _globals["module_level_client"] = sync_client + return sync_client + + raise AttributeError(f"HTTP handlers lazy import: unknown attribute {name!r}") \ No newline at end of file diff --git a/tests/test_litellm/test_lazy_imports.py b/tests/test_litellm/test_lazy_imports.py index e36acf22eb8..42060579842 100644 --- a/tests/test_litellm/test_lazy_imports.py +++ b/tests/test_litellm/test_lazy_imports.py @@ -12,9 +12,11 @@ from litellm._lazy_imports import ( COST_CALCULATOR_NAMES, LITELLM_LOGGING_NAMES, UTILS_NAMES, + HTTP_HANDLER_NAMES, _lazy_import_cost_calculator, _lazy_import_litellm_logging, _lazy_import_utils, + _lazy_import_http_handlers, ) @@ -78,6 +80,18 @@ def test_utils_lazy_imports(): _verify_only_requested_name_imported(name, UTILS_NAMES) +def test_http_handler_lazy_imports(): + """Test that HTTP handler singletons can be lazy imported.""" + for name in HTTP_HANDLER_NAMES: + _clear_names_from_globals(HTTP_HANDLER_NAMES) + + handler = _lazy_import_http_handlers(name) + assert handler is not None + assert name in litellm.__dict__ + + _verify_only_requested_name_imported(name, HTTP_HANDLER_NAMES) + + def test_unknown_attribute_raises_error(): """Test that unknown attributes raise AttributeError.""" with pytest.raises(AttributeError):