mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
Merge pull request #39969 from BerriAI/revert-39121-litellm_lazy_sdk_import
revert: perf: lazy-load SDK symbols so import litellm stays under 60 MB RSS (#39121)
This commit is contained in:
yuneng-jiang
GitHub
commit
b1e2f5bc0b
5 changed files with 202 additions and 3094 deletions
|
|
@ -13,7 +13,6 @@ warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
|
|||
### INIT VARIABLES #########################
|
||||
import threading
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
|
||||
import dotenv as _dotenv
|
||||
|
|
@ -46,6 +45,8 @@ from typing import (
|
|||
TYPE_CHECKING,
|
||||
Union,
|
||||
)
|
||||
from litellm.types.integrations.datadog import DatadogInitParams
|
||||
from litellm.types.integrations.newrelic import NewRelicInitParams
|
||||
from litellm._logging import (
|
||||
set_verbose,
|
||||
_turn_on_debug,
|
||||
|
|
@ -94,7 +95,8 @@ from litellm.constants import (
|
|||
DEFAULT_SOFT_BUDGET,
|
||||
DEFAULT_ALLOWED_FAILS,
|
||||
)
|
||||
# httpx is lazy-loaded via __getattr__
|
||||
import httpx
|
||||
|
||||
# register_async_client_cleanup is lazy-loaded and called on first access
|
||||
|
||||
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
|
||||
|
|
@ -362,6 +364,8 @@ guardrail_name_config_map: Dict[str, GuardrailItem] = {}
|
|||
include_cost_in_streaming_usage: bool = False
|
||||
reasoning_auto_summary: bool = False
|
||||
### PROMPTS ####
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
|
||||
prompt_name_config_map: Dict[str, PromptSpec] = {}
|
||||
|
||||
##################
|
||||
|
|
@ -1267,203 +1271,206 @@ openai_video_generation_models = ["sora-2"]
|
|||
# get_llm_provider is lazy-loaded via __getattr__
|
||||
# remove_index_from_tool_calls is lazy-loaded via __getattr__
|
||||
|
||||
# SDK symbols previously imported eagerly here are lazy-loaded via __getattr__
|
||||
# (_SDK_SYMBOLS_IMPORT_MAP in _lazy_imports_registry.py); mirrored under TYPE_CHECKING
|
||||
# so static type checkers still see them
|
||||
if TYPE_CHECKING:
|
||||
_key_management_settings: KeyManagementSettings
|
||||
# Import KeyManagementSettings here (before utils import) because _key_management_settings
|
||||
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
|
||||
from litellm.types.secret_managers.main import KeyManagementSettings
|
||||
|
||||
from .utils import client
|
||||
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
|
||||
|
||||
from .llms.custom_llm import CustomLLM
|
||||
from .llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
|
||||
from .llms.deprecated_providers.palm import (
|
||||
PalmConfig,
|
||||
) # here to prevent breaking changes
|
||||
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
|
||||
from .llms.gemini.common_utils import GeminiModelInfo
|
||||
# client must be imported immediately as it's used as a decorator at function definition time
|
||||
from .utils import client
|
||||
|
||||
from .llms.vertex_ai.vertex_embeddings.transformation import (
|
||||
VertexAITextEmbeddingConfig,
|
||||
)
|
||||
# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
|
||||
# (which imports tiktoken) at import time
|
||||
|
||||
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
|
||||
from .llms.custom_llm import CustomLLM
|
||||
from .llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
|
||||
from .llms.deprecated_providers.palm import (
|
||||
PalmConfig,
|
||||
) # here to prevent breaking changes
|
||||
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
|
||||
from .llms.gemini.common_utils import GeminiModelInfo
|
||||
|
||||
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
|
||||
AmazonTitanV2Config,
|
||||
)
|
||||
from .llms.topaz.common_utils import TopazModelInfo
|
||||
|
||||
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
|
||||
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
|
||||
from .llms.xai.common_utils import XAIModelInfo
|
||||
from .llms.vertex_ai.vertex_embeddings.transformation import (
|
||||
VertexAITextEmbeddingConfig,
|
||||
)
|
||||
|
||||
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
|
||||
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
|
||||
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
|
||||
|
||||
# Import LlmProviders here (before main import) because it's imported during import time
|
||||
# in multiple places including openai.py (via main import)
|
||||
|
||||
## Lazy loading this is not straightforward, will leave it here for now.
|
||||
from .main import *
|
||||
from .compression import compress
|
||||
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
|
||||
AmazonTitanV2Config,
|
||||
)
|
||||
from .llms.topaz.common_utils import TopazModelInfo
|
||||
|
||||
# Skills API
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
list_skills,
|
||||
alist_skills,
|
||||
get_skill,
|
||||
aget_skill,
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .evals.main import (
|
||||
create_eval,
|
||||
acreate_eval,
|
||||
list_evals,
|
||||
alist_evals,
|
||||
get_eval,
|
||||
aget_eval,
|
||||
delete_eval,
|
||||
adelete_eval,
|
||||
cancel_eval,
|
||||
acancel_eval,
|
||||
create_run,
|
||||
acreate_run,
|
||||
list_runs,
|
||||
alist_runs,
|
||||
get_run,
|
||||
aget_run,
|
||||
delete_run,
|
||||
adelete_run,
|
||||
cancel_run,
|
||||
acancel_run,
|
||||
)
|
||||
from .integrations import *
|
||||
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
||||
from .exceptions import (
|
||||
AuthenticationError,
|
||||
InvalidRequestError,
|
||||
BadRequestError,
|
||||
ImageFetchError,
|
||||
NotFoundError,
|
||||
PermissionDeniedError,
|
||||
RateLimitError,
|
||||
RateLimitErrorCategory,
|
||||
RateLimitType,
|
||||
ServiceUnavailableError,
|
||||
BadGatewayError,
|
||||
OpenAIError,
|
||||
ContextWindowExceededError,
|
||||
ContentPolicyViolationError,
|
||||
BudgetExceededError,
|
||||
APIError,
|
||||
Timeout,
|
||||
APIConnectionError,
|
||||
UnsupportedParamsError,
|
||||
APIResponseValidationError,
|
||||
UnprocessableEntityError,
|
||||
InternalServerError,
|
||||
JSONSchemaValidationError,
|
||||
LITELLM_EXCEPTION_TYPES,
|
||||
MockException,
|
||||
)
|
||||
from .budget_manager import BudgetManager
|
||||
from .proxy.proxy_cli import run_server
|
||||
from .router import Router
|
||||
from .assistants.main import *
|
||||
from .batches.main import *
|
||||
from .images.main import *
|
||||
from .videos.main import *
|
||||
from .batch_completion.main import *
|
||||
from .rerank_api.main import *
|
||||
from .llms.anthropic.experimental_pass_through.messages.handler import *
|
||||
from .responses.main import *
|
||||
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
|
||||
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
|
||||
from .llms.xai.common_utils import XAIModelInfo
|
||||
|
||||
# Interactions API is available as litellm.interactions module
|
||||
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
|
||||
from . import interactions
|
||||
from .interactions.agents.main import (
|
||||
acreate as acreate_agent,
|
||||
create as create_agent,
|
||||
alist as alist_agents,
|
||||
list as list_agents,
|
||||
aget as aget_agent,
|
||||
get as get_agent,
|
||||
adelete as adelete_agent,
|
||||
delete as delete_agent,
|
||||
alist_versions as alist_agent_versions,
|
||||
list_versions as list_agent_versions,
|
||||
)
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
list_skills,
|
||||
alist_skills,
|
||||
get_skill,
|
||||
aget_skill,
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .containers.main import *
|
||||
from .ocr.main import *
|
||||
from .rust_bridge import rust
|
||||
from .rag.main import *
|
||||
from .sandbox.main import *
|
||||
from .search.main import *
|
||||
from .realtime_api.main import (
|
||||
_arealtime,
|
||||
acreate_realtime_client_secret,
|
||||
acreate_realtime_transcription_session,
|
||||
arealtime_calls,
|
||||
)
|
||||
from .responses.main import _aresponses_websocket
|
||||
from .fine_tuning.main import *
|
||||
from .files.main import *
|
||||
from .vector_store_files.main import (
|
||||
acreate as avector_store_file_create,
|
||||
adelete as avector_store_file_delete,
|
||||
alist as avector_store_file_list,
|
||||
aretrieve as avector_store_file_retrieve,
|
||||
aretrieve_content as avector_store_file_content,
|
||||
aupdate as avector_store_file_update,
|
||||
create as vector_store_file_create,
|
||||
delete as vector_store_file_delete,
|
||||
list as vector_store_file_list,
|
||||
retrieve as vector_store_file_retrieve,
|
||||
retrieve_content as vector_store_file_content,
|
||||
update as vector_store_file_update,
|
||||
)
|
||||
from .scheduler import *
|
||||
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
|
||||
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
|
||||
|
||||
### ADAPTERS ###
|
||||
import litellm.anthropic_interface as anthropic
|
||||
# Import LlmProviders here (before main import) because it's imported during import time
|
||||
# in multiple places including openai.py (via main import)
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
### Vector Store Registry ###
|
||||
## Lazy loading this is not straightforward, will leave it here for now.
|
||||
from .main import *
|
||||
from .compression import compress
|
||||
|
||||
### RAG ###
|
||||
from . import rag
|
||||
# Skills API
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
list_skills,
|
||||
alist_skills,
|
||||
get_skill,
|
||||
aget_skill,
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .evals.main import (
|
||||
create_eval,
|
||||
acreate_eval,
|
||||
list_evals,
|
||||
alist_evals,
|
||||
get_eval,
|
||||
aget_eval,
|
||||
delete_eval,
|
||||
adelete_eval,
|
||||
cancel_eval,
|
||||
acancel_eval,
|
||||
create_run,
|
||||
acreate_run,
|
||||
list_runs,
|
||||
alist_runs,
|
||||
get_run,
|
||||
aget_run,
|
||||
delete_run,
|
||||
adelete_run,
|
||||
cancel_run,
|
||||
acancel_run,
|
||||
)
|
||||
from .integrations import *
|
||||
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
||||
from .exceptions import (
|
||||
AuthenticationError,
|
||||
InvalidRequestError,
|
||||
BadRequestError,
|
||||
ImageFetchError,
|
||||
NotFoundError,
|
||||
PermissionDeniedError,
|
||||
RateLimitError,
|
||||
RateLimitErrorCategory,
|
||||
RateLimitType,
|
||||
ServiceUnavailableError,
|
||||
BadGatewayError,
|
||||
OpenAIError,
|
||||
ContextWindowExceededError,
|
||||
ContentPolicyViolationError,
|
||||
BudgetExceededError,
|
||||
APIError,
|
||||
Timeout,
|
||||
APIConnectionError,
|
||||
UnsupportedParamsError,
|
||||
APIResponseValidationError,
|
||||
UnprocessableEntityError,
|
||||
InternalServerError,
|
||||
JSONSchemaValidationError,
|
||||
LITELLM_EXCEPTION_TYPES,
|
||||
MockException,
|
||||
)
|
||||
from .budget_manager import BudgetManager
|
||||
from .proxy.proxy_cli import run_server
|
||||
from .router import Router
|
||||
from .assistants.main import *
|
||||
from .batches.main import *
|
||||
from .images.main import *
|
||||
from .videos.main import *
|
||||
from .batch_completion.main import *
|
||||
from .rerank_api.main import *
|
||||
from .llms.anthropic.experimental_pass_through.messages.handler import *
|
||||
from .responses.main import *
|
||||
|
||||
### CUSTOM LLMs ###
|
||||
|
||||
### CLI UTILITIES ###
|
||||
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
|
||||
|
||||
### PASSTHROUGH ###
|
||||
from .passthrough import allm_passthrough_route, llm_passthrough_route
|
||||
from .google_genai import agenerate_content
|
||||
# Interactions API is available as litellm.interactions module
|
||||
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
|
||||
from . import interactions
|
||||
from .interactions.agents.main import (
|
||||
acreate as acreate_agent,
|
||||
create as create_agent,
|
||||
alist as alist_agents,
|
||||
list as list_agents,
|
||||
aget as aget_agent,
|
||||
get as get_agent,
|
||||
adelete as adelete_agent,
|
||||
delete as delete_agent,
|
||||
alist_versions as alist_agent_versions,
|
||||
list_versions as list_agent_versions,
|
||||
)
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
list_skills,
|
||||
alist_skills,
|
||||
get_skill,
|
||||
aget_skill,
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .containers.main import *
|
||||
from .ocr.main import *
|
||||
from .rust_bridge import rust
|
||||
from .rag.main import *
|
||||
from .sandbox.main import *
|
||||
from .search.main import *
|
||||
from .realtime_api.main import (
|
||||
_arealtime,
|
||||
acreate_realtime_client_secret,
|
||||
acreate_realtime_transcription_session,
|
||||
arealtime_calls,
|
||||
)
|
||||
from .responses.main import _aresponses_websocket
|
||||
from .fine_tuning.main import *
|
||||
from .files.main import *
|
||||
from .vector_store_files.main import (
|
||||
acreate as avector_store_file_create,
|
||||
adelete as avector_store_file_delete,
|
||||
alist as avector_store_file_list,
|
||||
aretrieve as avector_store_file_retrieve,
|
||||
aretrieve_content as avector_store_file_content,
|
||||
aupdate as avector_store_file_update,
|
||||
create as vector_store_file_create,
|
||||
delete as vector_store_file_delete,
|
||||
list as vector_store_file_list,
|
||||
retrieve as vector_store_file_retrieve,
|
||||
retrieve_content as vector_store_file_content,
|
||||
update as vector_store_file_update,
|
||||
)
|
||||
from .scheduler import *
|
||||
|
||||
### ADAPTERS ###
|
||||
from .types.adapter import AdapterItem
|
||||
import litellm.anthropic_interface as anthropic
|
||||
|
||||
adapters: List[AdapterItem] = []
|
||||
|
||||
### Vector Store Registry ###
|
||||
from .vector_stores.vector_store_registry import (
|
||||
VectorStoreRegistry,
|
||||
VectorStoreIndexRegistry,
|
||||
)
|
||||
|
||||
vector_store_registry: Optional[VectorStoreRegistry] = None
|
||||
vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None
|
||||
|
||||
### RAG ###
|
||||
from . import rag
|
||||
|
||||
### CUSTOM LLMs ###
|
||||
from .types.llms.custom_llm import CustomLLMItem
|
||||
|
||||
custom_provider_map: List[CustomLLMItem] = []
|
||||
_custom_providers: List[str] = [] # internal helper util, used to track names of custom providers
|
||||
disable_hf_tokenizer_download: Optional[bool] = (
|
||||
|
|
@ -1471,6 +1478,13 @@ disable_hf_tokenizer_download: Optional[bool] = (
|
|||
)
|
||||
global_disable_no_log_param: bool = False
|
||||
|
||||
### CLI UTILITIES ###
|
||||
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
|
||||
|
||||
### PASSTHROUGH ###
|
||||
from .passthrough import allm_passthrough_route, llm_passthrough_route
|
||||
from .google_genai import agenerate_content
|
||||
|
||||
### GLOBAL CONFIG ###
|
||||
global_bitbucket_config: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
|
@ -1494,21 +1508,10 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None:
|
|||
# Lazy loading system for heavy modules to reduce initial import time and memory usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
|
||||
from litellm.types.utils import ModelInfo as _ModelInfoType
|
||||
from litellm.types.utils import PriorityReservationSettings
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.types.adapter import AdapterItem
|
||||
from litellm.types.integrations.datadog import DatadogInitParams
|
||||
from litellm.types.integrations.newrelic import NewRelicInitParams
|
||||
from litellm.types.llms.custom_llm import CustomLLMItem
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.vector_stores.vector_store_registry import (
|
||||
VectorStoreIndexRegistry,
|
||||
VectorStoreRegistry,
|
||||
)
|
||||
|
||||
# Type stubs for lazy-loaded configs to help mypy
|
||||
from .llms.bedrock.chat.converse_transformation import (
|
||||
|
|
@ -2184,6 +2187,16 @@ if TYPE_CHECKING:
|
|||
# Track if async client cleanup has been registered (for lazy loading)
|
||||
_async_client_cleanup_registered = False
|
||||
|
||||
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
|
||||
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
|
||||
# For now, this only affects encoding (tiktoken) as it was the only reported issue
|
||||
# See: https://github.com/BerriAI/litellm/issues/18659
|
||||
# This ensures encoding is initialized before VCR starts recording HTTP requests
|
||||
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
|
||||
# Load encoding at import time (pre-#18070 behavior)
|
||||
# This ensures encoding is initialized before VCR starts recording
|
||||
from .main import encoding
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""Lazy import handler with cached registry for improved performance."""
|
||||
|
|
@ -2263,8 +2276,6 @@ def __getattr__(name: str) -> Any:
|
|||
"openAIGPT5Config": "OpenAIGPT5Config",
|
||||
"nvidiaNimConfig": "NvidiaNimConfig",
|
||||
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
|
||||
"vertexAITextEmbeddingConfig": "VertexAITextEmbeddingConfig",
|
||||
"_key_management_settings": "KeyManagementSettings",
|
||||
}
|
||||
if name in _config_instances:
|
||||
from ._lazy_imports import get_litellm_globals
|
||||
|
|
@ -2382,30 +2393,7 @@ def __getattr__(name: str) -> Any:
|
|||
|
||||
return locals()[name]
|
||||
|
||||
from ._lazy_imports import lazy_import_litellm_submodule
|
||||
|
||||
submodule: Final = lazy_import_litellm_submodule(name)
|
||||
if submodule is not None:
|
||||
return submodule
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
from ._lazy_imports import LiteLLMModule
|
||||
from ._lazy_imports_registry import STAR_IMPORT_PUBLIC_NAMES
|
||||
|
||||
sys.modules[__name__].__class__ = LiteLLMModule
|
||||
|
||||
__all__ = list(STAR_IMPORT_PUBLIC_NAMES) # mutable-ok: star imports require __all__ to be a list of str
|
||||
|
||||
|
||||
# ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time
|
||||
|
||||
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
|
||||
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
|
||||
# For now, this only affects encoding (tiktoken) as it was the only reported issue
|
||||
# See: https://github.com/BerriAI/litellm/issues/18659
|
||||
# This ensures encoding is initialized before VCR starts recording HTTP requests
|
||||
# This block stays at the bottom so __getattr__ can resolve attributes main.py needs during its import
|
||||
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
|
||||
from .main import encoding
|
||||
|
|
|
|||
|
|
@ -16,10 +16,9 @@ until they're actually needed.
|
|||
"""
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import sys
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType, ModuleType
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
|
@ -35,8 +34,6 @@ from ._lazy_imports_registry import (
|
|||
_LITELLM_LOGGING_IMPORT_MAP,
|
||||
_LLM_CONFIGS_IMPORT_MAP,
|
||||
_LLM_PROVIDER_LOGIC_IMPORT_MAP,
|
||||
_SDK_MODULE_ALIASES,
|
||||
_SDK_SYMBOLS_IMPORT_MAP,
|
||||
_TOKEN_COUNTER_IMPORT_MAP,
|
||||
_TYPES_IMPORT_MAP,
|
||||
_TYPES_UTILS_IMPORT_MAP,
|
||||
|
|
@ -81,10 +78,7 @@ def _get_utils_globals() -> dict[str, object]:
|
|||
This is where we cache imported attributes so we don't import them twice.
|
||||
When you do `litellm.utils.some_function`, it gets stored in this dictionary.
|
||||
"""
|
||||
cached: Final = sys.modules.get("litellm.utils")
|
||||
if cached is not None:
|
||||
return cached.__dict__
|
||||
return importlib.import_module("litellm.utils").__dict__
|
||||
return sys.modules["litellm.utils"].__dict__
|
||||
|
||||
|
||||
def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None":
|
||||
|
|
@ -220,10 +214,6 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
|
|||
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic
|
||||
for name in UTILS_MODULE_NAMES:
|
||||
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils_module
|
||||
for name in _SDK_SYMBOLS_IMPORT_MAP:
|
||||
_LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_symbols)
|
||||
for name in _SDK_MODULE_ALIASES:
|
||||
_LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_module_alias)
|
||||
|
||||
return _LAZY_IMPORT_REGISTRY
|
||||
|
||||
|
|
@ -360,86 +350,6 @@ def _lazy_import_llm_provider_logic(name: str) -> object:
|
|||
return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic")
|
||||
|
||||
|
||||
def _lazy_import_sdk_symbols(name: str) -> object:
|
||||
"""Handler for SDK symbols previously imported eagerly at the bottom of litellm/__init__.py"""
|
||||
return _generic_lazy_import(name, _SDK_SYMBOLS_IMPORT_MAP, "SDK symbols")
|
||||
|
||||
|
||||
def _lazy_import_sdk_module_alias(name: str) -> object:
|
||||
"""Handler for litellm attributes that bind a module (e.g. litellm.anthropic)"""
|
||||
_globals: Final = get_litellm_globals()
|
||||
if name in _globals:
|
||||
return _globals[name]
|
||||
module: Final = importlib.import_module(_SDK_MODULE_ALIASES[name])
|
||||
_globals[name] = module # rebind-ok: caches the resolved module alias on the package
|
||||
return module
|
||||
|
||||
|
||||
_SHADOWABLE_SDK_FUNCTIONS: Final = MappingProxyType(
|
||||
{
|
||||
"batch_completion": ("litellm.batch_completion.main", "batch_completion"),
|
||||
"ocr": ("litellm.ocr.main", "ocr"),
|
||||
"responses": ("litellm.responses.main", "responses"),
|
||||
"search": ("litellm.search.main", "search"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _shadowable_function_property(name: str) -> property:
|
||||
"""Property keeping litellm.<name> bound to the SDK function even after the import
|
||||
machinery binds the identically named litellm.<name> subpackage onto the litellm module."""
|
||||
module_path, attr_name = _SHADOWABLE_SDK_FUNCTIONS[name]
|
||||
|
||||
def _get(module: ModuleType) -> object:
|
||||
stored: Final = module.__dict__.get(name)
|
||||
if stored is not None and not (isinstance(stored, ModuleType) and stored.__name__ == f"litellm.{name}"):
|
||||
return stored
|
||||
value: Final = _module_attribute(importlib.import_module(module_path), attr_name)
|
||||
module.__dict__[name] = value # rebind-ok: caches the resolved function on the litellm module
|
||||
return value
|
||||
|
||||
def _set(module: ModuleType, value: object) -> None:
|
||||
module.__dict__[name] = value # rebind-ok: property setter must store assignments on the module
|
||||
|
||||
return property(_get, _set)
|
||||
|
||||
|
||||
class LiteLLMModule(ModuleType):
|
||||
"""Module type installed on the litellm package so function names shadowed by
|
||||
same-named subpackages (litellm.responses, ...) keep resolving to the functions."""
|
||||
|
||||
batch_completion = _shadowable_function_property("batch_completion")
|
||||
ocr = _shadowable_function_property("ocr")
|
||||
responses = _shadowable_function_property("responses")
|
||||
search = _shadowable_function_property("search")
|
||||
|
||||
|
||||
def lazy_import_submodule(package: str, name: str) -> "ModuleType | None":
|
||||
"""Resolve <package>.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
|
||||
if name.startswith("__") or not name.isidentifier():
|
||||
return None
|
||||
qualified_name: Final = f"{package}.{name}"
|
||||
try:
|
||||
spec: Final = importlib.util.find_spec(qualified_name)
|
||||
except ModuleNotFoundError:
|
||||
return None
|
||||
if spec is None:
|
||||
return None
|
||||
try:
|
||||
module: Final = importlib.import_module(qualified_name)
|
||||
except ModuleNotFoundError as exc:
|
||||
if exc.name == qualified_name:
|
||||
return None
|
||||
raise
|
||||
sys.modules[package].__dict__[name] = module # rebind-ok: caches the resolved submodule on the package
|
||||
return module
|
||||
|
||||
|
||||
def lazy_import_litellm_submodule(name: str) -> "ModuleType | None":
|
||||
"""Resolve litellm.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
|
||||
return lazy_import_submodule("litellm", name)
|
||||
|
||||
|
||||
def _lazy_import_utils_module(name: str) -> object:
|
||||
"""
|
||||
Handler for utils module lazy imports.
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,11 +1 @@
|
|||
from types import ModuleType
|
||||
from typing import Final
|
||||
|
||||
|
||||
def __getattr__(name: str) -> ModuleType:
|
||||
from litellm._lazy_imports import lazy_import_submodule
|
||||
|
||||
submodule: Final = lazy_import_submodule(__name__, name)
|
||||
if submodule is not None:
|
||||
return submodule
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
from . import *
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
"""Simple tests for lazy import functionality."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
|
@ -10,10 +7,6 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm._lazy_imports import (
|
||||
_SDK_MODULE_ALIASES,
|
||||
_SDK_SYMBOLS_IMPORT_MAP,
|
||||
lazy_import_litellm_submodule,
|
||||
_lazy_import_sdk_symbols,
|
||||
COST_CALCULATOR_NAMES,
|
||||
LITELLM_LOGGING_NAMES,
|
||||
UTILS_NAMES,
|
||||
|
|
@ -353,83 +346,3 @@ def test_utils_module_lazy_imports():
|
|||
assert name in utils_globals
|
||||
|
||||
_verify_only_requested_name_imported_in_utils(name, UTILS_MODULE_NAMES)
|
||||
|
||||
|
||||
def test_sdk_symbols_lazy_imports():
|
||||
"""Every symbol previously imported eagerly in litellm/__init__.py resolves to the source module attribute."""
|
||||
for name, (module_path, attr_name) in _SDK_SYMBOLS_IMPORT_MAP.items():
|
||||
resolved = getattr(litellm, name)
|
||||
expected = getattr(importlib.import_module(module_path), attr_name)
|
||||
assert resolved is expected, f"litellm.{name} is not {module_path}.{attr_name}"
|
||||
|
||||
|
||||
def test_sdk_module_aliases():
|
||||
"""Module-valued attributes (litellm.anthropic, litellm.httpx, ...) resolve to the aliased modules."""
|
||||
for name, module_path in _SDK_MODULE_ALIASES.items():
|
||||
assert getattr(litellm, name) is importlib.import_module(module_path)
|
||||
|
||||
|
||||
def test_litellm_submodule_fallback():
|
||||
"""litellm.<submodule> attribute access resolves real submodules and returns None for unknown names."""
|
||||
assert lazy_import_litellm_submodule("budget_manager") is importlib.import_module("litellm.budget_manager")
|
||||
assert litellm.utils is importlib.import_module("litellm.utils")
|
||||
assert lazy_import_litellm_submodule("not_a_real_submodule") is None
|
||||
with pytest.raises(AttributeError):
|
||||
_ = litellm.not_a_real_attribute
|
||||
|
||||
|
||||
def test_missing_attribute_stays_attribute_error_when_find_spec_lies(monkeypatch):
|
||||
"""getattr(litellm, name, default) must not leak ModuleNotFoundError when find_spec is patched to always succeed."""
|
||||
monkeypatch.setattr(importlib.util, "find_spec", lambda name: object())
|
||||
assert getattr(litellm, "not_a_real_submodule", None) is None
|
||||
with pytest.raises(AttributeError):
|
||||
_ = litellm.not_a_real_attribute
|
||||
|
||||
|
||||
def test_proxy_private_submodule_resolves_in_fresh_process():
|
||||
"""litellm.proxy._types resolves without an eager proxy import (used by documentation checks)."""
|
||||
code = "import litellm\nprint(litellm.proxy._types.__name__)\n"
|
||||
result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert result.stdout.strip() == "litellm.proxy._types"
|
||||
|
||||
|
||||
def test_lazy_instances_are_singletons():
|
||||
"""Lazily created instances are cached, so repeated access returns the same object."""
|
||||
assert litellm._key_management_settings is litellm._key_management_settings
|
||||
assert litellm.vertexAITextEmbeddingConfig is litellm.vertexAITextEmbeddingConfig
|
||||
from litellm.types.secret_managers.main import KeyManagementSettings
|
||||
|
||||
assert isinstance(litellm._key_management_settings, KeyManagementSettings)
|
||||
|
||||
|
||||
def test_star_import_exports_public_api():
|
||||
"""`from litellm import *` keeps exporting the full public surface despite lazy loading."""
|
||||
code = (
|
||||
"from litellm import *\n"
|
||||
"import litellm\n"
|
||||
"missing = [n for n in litellm.__all__ if n not in dir()]\n"
|
||||
"assert not missing, missing[:20]\n"
|
||||
"assert callable(completion) and callable(Router)\n"
|
||||
)
|
||||
result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "linux", reason="reads /proc for RSS")
|
||||
def test_import_litellm_stays_lightweight():
|
||||
"""`import litellm` must not pull in the SDK/proxy heavyweights or blow up RSS (LIT-6607)."""
|
||||
code = (
|
||||
"import json, re, sys\n"
|
||||
"import litellm\n"
|
||||
"heavy = [m for m in ('litellm.main', 'litellm.utils', 'litellm.router', 'litellm.proxy.proxy_cli',\n"
|
||||
" 'tiktoken', 'fastapi', 'grpc', 'boto3') if m in sys.modules]\n"
|
||||
"with open('/proc/self/status') as f:\n"
|
||||
" rss_kb = int(re.search(r'VmRSS:\\s+(\\d+) kB', f.read()).group(1))\n"
|
||||
"print(json.dumps({'total': len(sys.modules), 'heavy': heavy, 'rss_mb': rss_kb / 1024}))\n"
|
||||
)
|
||||
result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True)
|
||||
stats = json.loads(result.stdout)
|
||||
assert stats["heavy"] == [], f"heavy modules imported eagerly: {stats['heavy']}"
|
||||
assert stats["total"] < 800, f"import litellm loaded {stats['total']} modules"
|
||||
assert stats["rss_mb"] < 75, f"import litellm used {stats['rss_mb']:.1f} MB RSS"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue