From 2cbcaf2abf463a137e9a7f164c305da15e6c9413 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 3 Jan 2026 13:34:17 -0800 Subject: [PATCH] refactor(utils): lazy load heavy imports to improve import time and memory usage (#18610) * refactor(utils): lazy load heavy imports to improve import time - Move BaseVectorStore, CredentialAccessor, and exception_mapping_utils imports to lazy loading via __getattr__ - Add _get_utils_globals() helper function following pattern from _lazy_imports.py - Refactor __getattr__ to use consistent caching pattern matching __init__.py - Update load_credentials_from_list to use lazy-loaded CredentialAccessor This reduces import time and memory usage by only loading these modules when they're actually accessed, not during module import. * refactor(utils): lazy load additional heavy imports to improve import time - Move get_llm_provider, _is_non_openai_azure_model to lazy loading - Move get_supported_openai_params to lazy loading - Move convert_dict_to_response functions (LiteLLMResponseObjectHandler, convert_to_model_response_object, etc.) to lazy loading - Move get_api_base and ResponseMetadata to lazy loading - Move _parse_content_for_reasoning to lazy loading - Update all internal usages to access via getattr(sys.modules[__name__], ...) This reduces import time and memory usage by only loading these modules when they're actually accessed, not during module import. * fix(utils): suppress PLR0915 linter warning for __getattr__ function The __getattr__ function intentionally has many statements to handle multiple lazy-loaded imports. Add noqa comment to suppress the warning. * fix(utils): add type stubs for lazy-loaded functions in TYPE_CHECKING block Add type imports and declarations in TYPE_CHECKING block to help mypy understand the types of lazy-loaded functions accessed via __getattr__. This follows the same pattern used in __init__.py for lazy-loaded items. --- litellm/utils.py | 249 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 209 insertions(+), 40 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 3dbeeb970a4..e5eb57b0712 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -71,9 +71,6 @@ from litellm.constants import ( OPENAI_EMBEDDING_PARAMS, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) -from litellm.integrations.vector_store_integrations.base_vector_store import ( - BaseVectorStore, -) # Import cached imports utilities from litellm.litellm_core_utils.cached_imports import ( @@ -86,48 +83,21 @@ from litellm.litellm_core_utils.core_helpers import ( map_finish_reason, process_response_headers, ) -from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dot_notation_indexing import ( delete_nested_value, is_nested_path, ) -from litellm.litellm_core_utils.exception_mapping_utils import ( - _get_response_headers, - exception_type, - get_error_message, -) from litellm.litellm_core_utils.get_litellm_params import ( _get_base_model_from_litellm_call_metadata, get_litellm_params, ) -from litellm.litellm_core_utils.get_llm_provider_logic import ( - _is_non_openai_azure_model, - get_llm_provider, -) -from litellm.litellm_core_utils.get_supported_openai_params import ( - get_supported_openai_params, -) from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( - LiteLLMResponseObjectHandler, - _handle_invalid_parallel_tool_calls, - convert_to_model_response_object, - convert_to_streaming_response, - convert_to_streaming_response_async, -) -from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( get_formatted_prompt, ) from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) -from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - ResponseMetadata, -) -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - _parse_content_for_reasoning, -) from litellm.litellm_core_utils.redact_messages import ( LiteLLMLoggingObject, redact_message_input_output_from_logging, @@ -346,6 +316,30 @@ if TYPE_CHECKING: from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.proxy._types import AllowedModelRegion + # Type stubs for lazy-loaded functions to help mypy understand their types + # These imports allow mypy to understand the types when these are accessed via __getattr__ + from litellm.litellm_core_utils.exception_mapping_utils import exception_type + from litellm.litellm_core_utils.get_llm_provider_logic import ( + _is_non_openai_azure_model, + get_llm_provider, + ) + from litellm.litellm_core_utils.get_supported_openai_params import ( + get_supported_openai_params, + ) + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + LiteLLMResponseObjectHandler, + _handle_invalid_parallel_tool_calls, + convert_to_model_response_object, + convert_to_streaming_response, + convert_to_streaming_response_async, + ) + from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + ResponseMetadata, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning, + ) from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig @@ -618,10 +612,22 @@ def get_applied_guardrails(kwargs: Dict[str, Any]) -> List[str]: return applied_guardrails +def _get_utils_globals() -> dict: + """ + Get the globals dictionary of the utils module. + + This is where we cache imported attributes so we don't import them twice. + """ + return sys.modules[__name__].__dict__ + + def load_credentials_from_list(kwargs: dict): """ Updates kwargs with the credentials if credential_name in kwarg """ + # Access CredentialAccessor via module to trigger lazy loading if needed + CredentialAccessor = getattr(sys.modules[__name__], 'CredentialAccessor') + credential_name = kwargs.get("litellm_credential_name") if credential_name and litellm.credential_list: credential_accessor = CredentialAccessor.get_credential_values(credential_name) @@ -2259,6 +2265,7 @@ def supports_response_schema( """ ## GET LLM PROVIDER ## try: + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') model, custom_llm_provider, _, _ = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider ) @@ -2956,6 +2963,9 @@ def get_optional_params_embeddings( # noqa: PLR0915 additional_drop_params: Optional[List[str]] = None, **kwargs, ): + # Lazy load get_supported_openai_params + get_supported_openai_params = getattr(sys.modules[__name__], 'get_supported_openai_params') + # retrieve all parameters passed to the function passed_params = locals() custom_llm_provider = passed_params.pop("custom_llm_provider", None) @@ -3758,6 +3768,7 @@ def get_optional_params( # noqa: PLR0915 message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.", ) + get_supported_openai_params = getattr(sys.modules[__name__], 'get_supported_openai_params') supported_params = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider ) @@ -4895,6 +4906,7 @@ def get_max_tokens(model: str) -> Optional[int]: return litellm.model_cost[model]["max_output_tokens"] elif "max_tokens" in litellm.model_cost[model]: return litellm.model_cost[model]["max_tokens"] + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') model, custom_llm_provider, _, _ = get_llm_provider(model=model) if custom_llm_provider == "huggingface": max_tokens = _get_max_position_embeddings(model_name=model) @@ -5015,6 +5027,7 @@ def _get_potential_model_names( if custom_llm_provider is None: # Get custom_llm_provider try: + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') split_model, custom_llm_provider, _, _ = get_llm_provider(model=model) except Exception: split_model = model @@ -5737,6 +5750,7 @@ def validate_environment( # noqa: PLR0915 } ## EXTRACT LLM PROVIDER - if model name provided try: + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') _, custom_llm_provider, _, _ = get_llm_provider(model=model) except Exception: custom_llm_provider = None @@ -6299,6 +6313,7 @@ def register_prompt_template( complete_model = model potential_models = [complete_model] try: + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') model = get_llm_provider(model=model)[0] potential_models.append(model) except Exception: @@ -6384,6 +6399,7 @@ class TextCompletionStreamWrapper: except StopIteration: raise StopIteration except Exception as e: + exception_type = getattr(sys.modules[__name__], 'exception_type') raise exception_type( model=self.model, custom_llm_provider=self.custom_llm_provider or "", @@ -8705,16 +8721,169 @@ def should_run_mock_completion( return False -# Re-export encoding from main.py for backward compatibility -# This allows tests to import: from litellm.utils import encoding -# We use a lazy import to avoid loading main.py at utils.py import time -def __getattr__(name: str) -> Any: +def __getattr__(name: str) -> Any: # noqa: PLR0915 """Lazy import handler for utils module""" + _globals = _get_utils_globals() + + # Lazy load encoding from main.py to avoid heavy tiktoken import if name == "encoding": - # Cache it in the module's __dict__ for subsequent accesses - import sys - - from litellm.main import encoding as _encoding - sys.modules[__name__].__dict__["encoding"] = _encoding - return _encoding + # Check if already cached + if "encoding" not in _globals: + from litellm.main import encoding as _encoding + _globals["encoding"] = _encoding + return _globals["encoding"] + + # Lazy load BaseVectorStore to avoid loading it at module import time + if name == "BaseVectorStore": + # Check if already cached + if "BaseVectorStore" not in _globals: + from litellm.integrations.vector_store_integrations.base_vector_store import ( + BaseVectorStore as _BaseVectorStore, + ) + _globals["BaseVectorStore"] = _BaseVectorStore + return _globals["BaseVectorStore"] + + # Lazy load CredentialAccessor to avoid loading it at module import time + if name == "CredentialAccessor": + # Check if already cached + if "CredentialAccessor" not in _globals: + from litellm.litellm_core_utils.credential_accessor import ( + CredentialAccessor as _CredentialAccessor, + ) + _globals["CredentialAccessor"] = _CredentialAccessor + return _globals["CredentialAccessor"] + + # Lazy load exception_mapping_utils functions to avoid loading at module import time + if name == "exception_type": + # Check if already cached + if "exception_type" not in _globals: + from litellm.litellm_core_utils.exception_mapping_utils import ( + exception_type as _exception_type, + ) + _globals["exception_type"] = _exception_type + return _globals["exception_type"] + + if name == "get_error_message": + # Check if already cached + if "get_error_message" not in _globals: + from litellm.litellm_core_utils.exception_mapping_utils import ( + get_error_message as _get_error_message, + ) + _globals["get_error_message"] = _get_error_message + return _globals["get_error_message"] + + if name == "_get_response_headers": + # Check if already cached + if "_get_response_headers" not in _globals: + from litellm.litellm_core_utils.exception_mapping_utils import ( + _get_response_headers as __get_response_headers, + ) + _globals["_get_response_headers"] = __get_response_headers + return _globals["_get_response_headers"] + + # Lazy load get_llm_provider_logic functions to avoid loading at module import time + if name == "get_llm_provider": + # Check if already cached + if "get_llm_provider" not in _globals: + from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider as _get_llm_provider, + ) + _globals["get_llm_provider"] = _get_llm_provider + return _globals["get_llm_provider"] + + if name == "_is_non_openai_azure_model": + # Check if already cached + if "_is_non_openai_azure_model" not in _globals: + from litellm.litellm_core_utils.get_llm_provider_logic import ( + _is_non_openai_azure_model as __is_non_openai_azure_model, + ) + _globals["_is_non_openai_azure_model"] = __is_non_openai_azure_model + return _globals["_is_non_openai_azure_model"] + + # Lazy load get_supported_openai_params to avoid loading at module import time + if name == "get_supported_openai_params": + # Check if already cached + if "get_supported_openai_params" not in _globals: + from litellm.litellm_core_utils.get_supported_openai_params import ( + get_supported_openai_params as _get_supported_openai_params, + ) + _globals["get_supported_openai_params"] = _get_supported_openai_params + return _globals["get_supported_openai_params"] + + # Lazy load convert_dict_to_response functions to avoid loading at module import time + if name == "LiteLLMResponseObjectHandler": + # Check if already cached + if "LiteLLMResponseObjectHandler" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + LiteLLMResponseObjectHandler as _LiteLLMResponseObjectHandler, + ) + _globals["LiteLLMResponseObjectHandler"] = _LiteLLMResponseObjectHandler + return _globals["LiteLLMResponseObjectHandler"] + + if name == "_handle_invalid_parallel_tool_calls": + # Check if already cached + if "_handle_invalid_parallel_tool_calls" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _handle_invalid_parallel_tool_calls as __handle_invalid_parallel_tool_calls, + ) + _globals["_handle_invalid_parallel_tool_calls"] = __handle_invalid_parallel_tool_calls + return _globals["_handle_invalid_parallel_tool_calls"] + + if name == "convert_to_model_response_object": + # Check if already cached + if "convert_to_model_response_object" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_model_response_object as _convert_to_model_response_object, + ) + _globals["convert_to_model_response_object"] = _convert_to_model_response_object + return _globals["convert_to_model_response_object"] + + if name == "convert_to_streaming_response": + # Check if already cached + if "convert_to_streaming_response" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response as _convert_to_streaming_response, + ) + _globals["convert_to_streaming_response"] = _convert_to_streaming_response + return _globals["convert_to_streaming_response"] + + if name == "convert_to_streaming_response_async": + # Check if already cached + if "convert_to_streaming_response_async" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response_async as _convert_to_streaming_response_async, + ) + _globals["convert_to_streaming_response_async"] = _convert_to_streaming_response_async + return _globals["convert_to_streaming_response_async"] + + # Lazy load get_api_base to avoid loading at module import time + if name == "get_api_base": + # Check if already cached + if "get_api_base" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.get_api_base import ( + get_api_base as _get_api_base, + ) + _globals["get_api_base"] = _get_api_base + return _globals["get_api_base"] + + # Lazy load ResponseMetadata to avoid loading at module import time + if name == "ResponseMetadata": + # Check if already cached + if "ResponseMetadata" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + ResponseMetadata as _ResponseMetadata, + ) + _globals["ResponseMetadata"] = _ResponseMetadata + return _globals["ResponseMetadata"] + + # Lazy load _parse_content_for_reasoning to avoid loading at module import time + if name == "_parse_content_for_reasoning": + # Check if already cached + if "_parse_content_for_reasoning" not in _globals: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning as __parse_content_for_reasoning, + ) + _globals["_parse_content_for_reasoning"] = __parse_content_for_reasoning + return _globals["_parse_content_for_reasoning"] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")