mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-07 08:26:10 +00:00
refactor(utils): lazy load heavy imports to improve import time and memory usage (#18610)
* refactor(utils): lazy load heavy imports to improve import time - Move BaseVectorStore, CredentialAccessor, and exception_mapping_utils imports to lazy loading via __getattr__ - Add _get_utils_globals() helper function following pattern from _lazy_imports.py - Refactor __getattr__ to use consistent caching pattern matching __init__.py - Update load_credentials_from_list to use lazy-loaded CredentialAccessor This reduces import time and memory usage by only loading these modules when they're actually accessed, not during module import. * refactor(utils): lazy load additional heavy imports to improve import time - Move get_llm_provider, _is_non_openai_azure_model to lazy loading - Move get_supported_openai_params to lazy loading - Move convert_dict_to_response functions (LiteLLMResponseObjectHandler, convert_to_model_response_object, etc.) to lazy loading - Move get_api_base and ResponseMetadata to lazy loading - Move _parse_content_for_reasoning to lazy loading - Update all internal usages to access via getattr(sys.modules[__name__], ...) This reduces import time and memory usage by only loading these modules when they're actually accessed, not during module import. * fix(utils): suppress PLR0915 linter warning for __getattr__ function The __getattr__ function intentionally has many statements to handle multiple lazy-loaded imports. Add noqa comment to suppress the warning. * fix(utils): add type stubs for lazy-loaded functions in TYPE_CHECKING block Add type imports and declarations in TYPE_CHECKING block to help mypy understand the types of lazy-loaded functions accessed via __getattr__. This follows the same pattern used in __init__.py for lazy-loaded items.
This commit is contained in:
Alexsander Hamir
GitHub
parent
d0b030f753
commit
2cbcaf2abf
1 changed files with 209 additions and 40 deletions
249
litellm/utils.py
249
litellm/utils.py
|
|
@ -71,9 +71,6 @@ from litellm.constants import (
|
|||
OPENAI_EMBEDDING_PARAMS,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
from litellm.integrations.vector_store_integrations.base_vector_store import (
|
||||
BaseVectorStore,
|
||||
)
|
||||
|
||||
# Import cached imports utilities
|
||||
from litellm.litellm_core_utils.cached_imports import (
|
||||
|
|
@ -86,48 +83,21 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
map_finish_reason,
|
||||
process_response_headers,
|
||||
)
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import (
|
||||
delete_nested_value,
|
||||
is_nested_path,
|
||||
)
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import (
|
||||
_get_response_headers,
|
||||
exception_type,
|
||||
get_error_message,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_litellm_params import (
|
||||
_get_base_model_from_litellm_call_metadata,
|
||||
get_litellm_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
_is_non_openai_azure_model,
|
||||
get_llm_provider,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
LiteLLMResponseObjectHandler,
|
||||
_handle_invalid_parallel_tool_calls,
|
||||
convert_to_model_response_object,
|
||||
convert_to_streaming_response,
|
||||
convert_to_streaming_response_async,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import (
|
||||
get_formatted_prompt,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
||||
get_response_headers,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
||||
ResponseMetadata,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_parse_content_for_reasoning,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
LiteLLMLoggingObject,
|
||||
redact_message_input_output_from_logging,
|
||||
|
|
@ -346,6 +316,30 @@ if TYPE_CHECKING:
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.base_llm.files.transformation import BaseFilesConfig
|
||||
from litellm.proxy._types import AllowedModelRegion
|
||||
# Type stubs for lazy-loaded functions to help mypy understand their types
|
||||
# These imports allow mypy to understand the types when these are accessed via __getattr__
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
_is_non_openai_azure_model,
|
||||
get_llm_provider,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
LiteLLMResponseObjectHandler,
|
||||
_handle_invalid_parallel_tool_calls,
|
||||
convert_to_model_response_object,
|
||||
convert_to_streaming_response,
|
||||
convert_to_streaming_response_async,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
||||
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
||||
ResponseMetadata,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_parse_content_for_reasoning,
|
||||
)
|
||||
|
||||
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig
|
||||
|
|
@ -618,10 +612,22 @@ def get_applied_guardrails(kwargs: Dict[str, Any]) -> List[str]:
|
|||
return applied_guardrails
|
||||
|
||||
|
||||
def _get_utils_globals() -> dict:
|
||||
"""
|
||||
Get the globals dictionary of the utils module.
|
||||
|
||||
This is where we cache imported attributes so we don't import them twice.
|
||||
"""
|
||||
return sys.modules[__name__].__dict__
|
||||
|
||||
|
||||
def load_credentials_from_list(kwargs: dict):
|
||||
"""
|
||||
Updates kwargs with the credentials if credential_name in kwarg
|
||||
"""
|
||||
# Access CredentialAccessor via module to trigger lazy loading if needed
|
||||
CredentialAccessor = getattr(sys.modules[__name__], 'CredentialAccessor')
|
||||
|
||||
credential_name = kwargs.get("litellm_credential_name")
|
||||
if credential_name and litellm.credential_list:
|
||||
credential_accessor = CredentialAccessor.get_credential_values(credential_name)
|
||||
|
|
@ -2259,6 +2265,7 @@ def supports_response_schema(
|
|||
"""
|
||||
## GET LLM PROVIDER ##
|
||||
try:
|
||||
get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
|
||||
model, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
|
@ -2956,6 +2963,9 @@ def get_optional_params_embeddings( # noqa: PLR0915
|
|||
additional_drop_params: Optional[List[str]] = None,
|
||||
**kwargs,
|
||||
):
|
||||
# Lazy load get_supported_openai_params
|
||||
get_supported_openai_params = getattr(sys.modules[__name__], 'get_supported_openai_params')
|
||||
|
||||
# retrieve all parameters passed to the function
|
||||
passed_params = locals()
|
||||
custom_llm_provider = passed_params.pop("custom_llm_provider", None)
|
||||
|
|
@ -3758,6 +3768,7 @@ def get_optional_params( # noqa: PLR0915
|
|||
message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.",
|
||||
)
|
||||
|
||||
get_supported_openai_params = getattr(sys.modules[__name__], 'get_supported_openai_params')
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
)
|
||||
|
|
@ -4895,6 +4906,7 @@ def get_max_tokens(model: str) -> Optional[int]:
|
|||
return litellm.model_cost[model]["max_output_tokens"]
|
||||
elif "max_tokens" in litellm.model_cost[model]:
|
||||
return litellm.model_cost[model]["max_tokens"]
|
||||
get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
|
||||
model, custom_llm_provider, _, _ = get_llm_provider(model=model)
|
||||
if custom_llm_provider == "huggingface":
|
||||
max_tokens = _get_max_position_embeddings(model_name=model)
|
||||
|
|
@ -5015,6 +5027,7 @@ def _get_potential_model_names(
|
|||
if custom_llm_provider is None:
|
||||
# Get custom_llm_provider
|
||||
try:
|
||||
get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
|
||||
split_model, custom_llm_provider, _, _ = get_llm_provider(model=model)
|
||||
except Exception:
|
||||
split_model = model
|
||||
|
|
@ -5737,6 +5750,7 @@ def validate_environment( # noqa: PLR0915
|
|||
}
|
||||
## EXTRACT LLM PROVIDER - if model name provided
|
||||
try:
|
||||
get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
|
||||
_, custom_llm_provider, _, _ = get_llm_provider(model=model)
|
||||
except Exception:
|
||||
custom_llm_provider = None
|
||||
|
|
@ -6299,6 +6313,7 @@ def register_prompt_template(
|
|||
complete_model = model
|
||||
potential_models = [complete_model]
|
||||
try:
|
||||
get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider')
|
||||
model = get_llm_provider(model=model)[0]
|
||||
potential_models.append(model)
|
||||
except Exception:
|
||||
|
|
@ -6384,6 +6399,7 @@ class TextCompletionStreamWrapper:
|
|||
except StopIteration:
|
||||
raise StopIteration
|
||||
except Exception as e:
|
||||
exception_type = getattr(sys.modules[__name__], 'exception_type')
|
||||
raise exception_type(
|
||||
model=self.model,
|
||||
custom_llm_provider=self.custom_llm_provider or "",
|
||||
|
|
@ -8705,16 +8721,169 @@ def should_run_mock_completion(
|
|||
return False
|
||||
|
||||
|
||||
# Re-export encoding from main.py for backward compatibility
|
||||
# This allows tests to import: from litellm.utils import encoding
|
||||
# We use a lazy import to avoid loading main.py at utils.py import time
|
||||
def __getattr__(name: str) -> Any:
|
||||
def __getattr__(name: str) -> Any: # noqa: PLR0915
|
||||
"""Lazy import handler for utils module"""
|
||||
_globals = _get_utils_globals()
|
||||
|
||||
# Lazy load encoding from main.py to avoid heavy tiktoken import
|
||||
if name == "encoding":
|
||||
# Cache it in the module's __dict__ for subsequent accesses
|
||||
import sys
|
||||
|
||||
from litellm.main import encoding as _encoding
|
||||
sys.modules[__name__].__dict__["encoding"] = _encoding
|
||||
return _encoding
|
||||
# Check if already cached
|
||||
if "encoding" not in _globals:
|
||||
from litellm.main import encoding as _encoding
|
||||
_globals["encoding"] = _encoding
|
||||
return _globals["encoding"]
|
||||
|
||||
# Lazy load BaseVectorStore to avoid loading it at module import time
|
||||
if name == "BaseVectorStore":
|
||||
# Check if already cached
|
||||
if "BaseVectorStore" not in _globals:
|
||||
from litellm.integrations.vector_store_integrations.base_vector_store import (
|
||||
BaseVectorStore as _BaseVectorStore,
|
||||
)
|
||||
_globals["BaseVectorStore"] = _BaseVectorStore
|
||||
return _globals["BaseVectorStore"]
|
||||
|
||||
# Lazy load CredentialAccessor to avoid loading it at module import time
|
||||
if name == "CredentialAccessor":
|
||||
# Check if already cached
|
||||
if "CredentialAccessor" not in _globals:
|
||||
from litellm.litellm_core_utils.credential_accessor import (
|
||||
CredentialAccessor as _CredentialAccessor,
|
||||
)
|
||||
_globals["CredentialAccessor"] = _CredentialAccessor
|
||||
return _globals["CredentialAccessor"]
|
||||
|
||||
# Lazy load exception_mapping_utils functions to avoid loading at module import time
|
||||
if name == "exception_type":
|
||||
# Check if already cached
|
||||
if "exception_type" not in _globals:
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import (
|
||||
exception_type as _exception_type,
|
||||
)
|
||||
_globals["exception_type"] = _exception_type
|
||||
return _globals["exception_type"]
|
||||
|
||||
if name == "get_error_message":
|
||||
# Check if already cached
|
||||
if "get_error_message" not in _globals:
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import (
|
||||
get_error_message as _get_error_message,
|
||||
)
|
||||
_globals["get_error_message"] = _get_error_message
|
||||
return _globals["get_error_message"]
|
||||
|
||||
if name == "_get_response_headers":
|
||||
# Check if already cached
|
||||
if "_get_response_headers" not in _globals:
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import (
|
||||
_get_response_headers as __get_response_headers,
|
||||
)
|
||||
_globals["_get_response_headers"] = __get_response_headers
|
||||
return _globals["_get_response_headers"]
|
||||
|
||||
# Lazy load get_llm_provider_logic functions to avoid loading at module import time
|
||||
if name == "get_llm_provider":
|
||||
# Check if already cached
|
||||
if "get_llm_provider" not in _globals:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
get_llm_provider as _get_llm_provider,
|
||||
)
|
||||
_globals["get_llm_provider"] = _get_llm_provider
|
||||
return _globals["get_llm_provider"]
|
||||
|
||||
if name == "_is_non_openai_azure_model":
|
||||
# Check if already cached
|
||||
if "_is_non_openai_azure_model" not in _globals:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
_is_non_openai_azure_model as __is_non_openai_azure_model,
|
||||
)
|
||||
_globals["_is_non_openai_azure_model"] = __is_non_openai_azure_model
|
||||
return _globals["_is_non_openai_azure_model"]
|
||||
|
||||
# Lazy load get_supported_openai_params to avoid loading at module import time
|
||||
if name == "get_supported_openai_params":
|
||||
# Check if already cached
|
||||
if "get_supported_openai_params" not in _globals:
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params as _get_supported_openai_params,
|
||||
)
|
||||
_globals["get_supported_openai_params"] = _get_supported_openai_params
|
||||
return _globals["get_supported_openai_params"]
|
||||
|
||||
# Lazy load convert_dict_to_response functions to avoid loading at module import time
|
||||
if name == "LiteLLMResponseObjectHandler":
|
||||
# Check if already cached
|
||||
if "LiteLLMResponseObjectHandler" not in _globals:
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
LiteLLMResponseObjectHandler as _LiteLLMResponseObjectHandler,
|
||||
)
|
||||
_globals["LiteLLMResponseObjectHandler"] = _LiteLLMResponseObjectHandler
|
||||
return _globals["LiteLLMResponseObjectHandler"]
|
||||
|
||||
if name == "_handle_invalid_parallel_tool_calls":
|
||||
# Check if already cached
|
||||
if "_handle_invalid_parallel_tool_calls" not in _globals:
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
_handle_invalid_parallel_tool_calls as __handle_invalid_parallel_tool_calls,
|
||||
)
|
||||
_globals["_handle_invalid_parallel_tool_calls"] = __handle_invalid_parallel_tool_calls
|
||||
return _globals["_handle_invalid_parallel_tool_calls"]
|
||||
|
||||
if name == "convert_to_model_response_object":
|
||||
# Check if already cached
|
||||
if "convert_to_model_response_object" not in _globals:
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
convert_to_model_response_object as _convert_to_model_response_object,
|
||||
)
|
||||
_globals["convert_to_model_response_object"] = _convert_to_model_response_object
|
||||
return _globals["convert_to_model_response_object"]
|
||||
|
||||
if name == "convert_to_streaming_response":
|
||||
# Check if already cached
|
||||
if "convert_to_streaming_response" not in _globals:
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
convert_to_streaming_response as _convert_to_streaming_response,
|
||||
)
|
||||
_globals["convert_to_streaming_response"] = _convert_to_streaming_response
|
||||
return _globals["convert_to_streaming_response"]
|
||||
|
||||
if name == "convert_to_streaming_response_async":
|
||||
# Check if already cached
|
||||
if "convert_to_streaming_response_async" not in _globals:
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
convert_to_streaming_response_async as _convert_to_streaming_response_async,
|
||||
)
|
||||
_globals["convert_to_streaming_response_async"] = _convert_to_streaming_response_async
|
||||
return _globals["convert_to_streaming_response_async"]
|
||||
|
||||
# Lazy load get_api_base to avoid loading at module import time
|
||||
if name == "get_api_base":
|
||||
# Check if already cached
|
||||
if "get_api_base" not in _globals:
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_api_base import (
|
||||
get_api_base as _get_api_base,
|
||||
)
|
||||
_globals["get_api_base"] = _get_api_base
|
||||
return _globals["get_api_base"]
|
||||
|
||||
# Lazy load ResponseMetadata to avoid loading at module import time
|
||||
if name == "ResponseMetadata":
|
||||
# Check if already cached
|
||||
if "ResponseMetadata" not in _globals:
|
||||
from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
||||
ResponseMetadata as _ResponseMetadata,
|
||||
)
|
||||
_globals["ResponseMetadata"] = _ResponseMetadata
|
||||
return _globals["ResponseMetadata"]
|
||||
|
||||
# Lazy load _parse_content_for_reasoning to avoid loading at module import time
|
||||
if name == "_parse_content_for_reasoning":
|
||||
# Check if already cached
|
||||
if "_parse_content_for_reasoning" not in _globals:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
_parse_content_for_reasoning as __parse_content_for_reasoning,
|
||||
)
|
||||
_globals["_parse_content_for_reasoning"] = __parse_content_for_reasoning
|
||||
return _globals["_parse_content_for_reasoning"]
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue