diff --git a/litellm/__init__.py b/litellm/__init__.py index b2bf3f09152..42c0ea881fd 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -13,7 +13,6 @@ warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*") ### INIT VARIABLES ######################### import threading import os -import sys # Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available import dotenv as _dotenv @@ -46,6 +45,8 @@ from typing import ( TYPE_CHECKING, Union, ) +from litellm.types.integrations.datadog import DatadogInitParams +from litellm.types.integrations.newrelic import NewRelicInitParams from litellm._logging import ( set_verbose, _turn_on_debug, @@ -94,7 +95,8 @@ from litellm.constants import ( DEFAULT_SOFT_BUDGET, DEFAULT_ALLOWED_FAILS, ) -# httpx is lazy-loaded via __getattr__ +import httpx + # register_async_client_cleanup is lazy-loaded and called on first access litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" @@ -362,6 +364,8 @@ guardrail_name_config_map: Dict[str, GuardrailItem] = {} include_cost_in_streaming_usage: bool = False reasoning_auto_summary: bool = False ### PROMPTS #### +from litellm.types.prompts.init_prompts import PromptSpec + prompt_name_config_map: Dict[str, PromptSpec] = {} ################## @@ -1267,203 +1271,206 @@ openai_video_generation_models = ["sora-2"] # get_llm_provider is lazy-loaded via __getattr__ # remove_index_from_tool_calls is lazy-loaded via __getattr__ -# SDK symbols previously imported eagerly here are lazy-loaded via __getattr__ -# (_SDK_SYMBOLS_IMPORT_MAP in _lazy_imports_registry.py); mirrored under TYPE_CHECKING -# so static type checkers still see them -if TYPE_CHECKING: - _key_management_settings: KeyManagementSettings +# Import KeyManagementSettings here (before utils import) because _key_management_settings +# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils) +from litellm.types.secret_managers.main import KeyManagementSettings - from .utils import client +_key_management_settings: KeyManagementSettings = KeyManagementSettings() - from .llms.custom_llm import CustomLLM - from .llms.anthropic.common_utils import AnthropicModelInfo - from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config - from .llms.deprecated_providers.palm import ( - PalmConfig, - ) # here to prevent breaking changes - from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig - from .llms.gemini.common_utils import GeminiModelInfo +# client must be imported immediately as it's used as a decorator at function definition time +from .utils import client - from .llms.vertex_ai.vertex_embeddings.transformation import ( - VertexAITextEmbeddingConfig, - ) +# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py +# (which imports tiktoken) at import time - vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig() +from .llms.custom_llm import CustomLLM +from .llms.anthropic.common_utils import AnthropicModelInfo +from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config +from .llms.deprecated_providers.palm import ( + PalmConfig, +) # here to prevent breaking changes +from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig +from .llms.gemini.common_utils import GeminiModelInfo - from .llms.bedrock.embed.amazon_titan_v2_transformation import ( - AmazonTitanV2Config, - ) - from .llms.topaz.common_utils import TopazModelInfo - # OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access - # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access - from .llms.xai.common_utils import XAIModelInfo +from .llms.vertex_ai.vertex_embeddings.transformation import ( + VertexAITextEmbeddingConfig, +) - # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) - # All remaining configs are now lazy loaded - see _lazy_imports_registry.py +vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig() - # Import LlmProviders here (before main import) because it's imported during import time - # in multiple places including openai.py (via main import) - ## Lazy loading this is not straightforward, will leave it here for now. - from .main import * - from .compression import compress +from .llms.bedrock.embed.amazon_titan_v2_transformation import ( + AmazonTitanV2Config, +) +from .llms.topaz.common_utils import TopazModelInfo - # Skills API - from .skills.main import ( - create_skill, - acreate_skill, - list_skills, - alist_skills, - get_skill, - aget_skill, - delete_skill, - adelete_skill, - ) - from .evals.main import ( - create_eval, - acreate_eval, - list_evals, - alist_evals, - get_eval, - aget_eval, - delete_eval, - adelete_eval, - cancel_eval, - acancel_eval, - create_run, - acreate_run, - list_runs, - alist_runs, - get_run, - aget_run, - delete_run, - adelete_run, - cancel_run, - acancel_run, - ) - from .integrations import * - from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients - from .exceptions import ( - AuthenticationError, - InvalidRequestError, - BadRequestError, - ImageFetchError, - NotFoundError, - PermissionDeniedError, - RateLimitError, - RateLimitErrorCategory, - RateLimitType, - ServiceUnavailableError, - BadGatewayError, - OpenAIError, - ContextWindowExceededError, - ContentPolicyViolationError, - BudgetExceededError, - APIError, - Timeout, - APIConnectionError, - UnsupportedParamsError, - APIResponseValidationError, - UnprocessableEntityError, - InternalServerError, - JSONSchemaValidationError, - LITELLM_EXCEPTION_TYPES, - MockException, - ) - from .budget_manager import BudgetManager - from .proxy.proxy_cli import run_server - from .router import Router - from .assistants.main import * - from .batches.main import * - from .images.main import * - from .videos.main import * - from .batch_completion.main import * - from .rerank_api.main import * - from .llms.anthropic.experimental_pass_through.messages.handler import * - from .responses.main import * +# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access +# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access +from .llms.xai.common_utils import XAIModelInfo - # Interactions API is available as litellm.interactions module - # Usage: litellm.interactions.create(), litellm.interactions.get(), etc. - from . import interactions - from .interactions.agents.main import ( - acreate as acreate_agent, - create as create_agent, - alist as alist_agents, - list as list_agents, - aget as aget_agent, - get as get_agent, - adelete as adelete_agent, - delete as delete_agent, - alist_versions as alist_agent_versions, - list_versions as list_agent_versions, - ) - from .skills.main import ( - create_skill, - acreate_skill, - list_skills, - alist_skills, - get_skill, - aget_skill, - delete_skill, - adelete_skill, - ) - from .containers.main import * - from .ocr.main import * - from .rust_bridge import rust - from .rag.main import * - from .sandbox.main import * - from .search.main import * - from .realtime_api.main import ( - _arealtime, - acreate_realtime_client_secret, - acreate_realtime_transcription_session, - arealtime_calls, - ) - from .responses.main import _aresponses_websocket - from .fine_tuning.main import * - from .files.main import * - from .vector_store_files.main import ( - acreate as avector_store_file_create, - adelete as avector_store_file_delete, - alist as avector_store_file_list, - aretrieve as avector_store_file_retrieve, - aretrieve_content as avector_store_file_content, - aupdate as avector_store_file_update, - create as vector_store_file_create, - delete as vector_store_file_delete, - list as vector_store_file_list, - retrieve as vector_store_file_retrieve, - retrieve_content as vector_store_file_content, - update as vector_store_file_update, - ) - from .scheduler import * +# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) +# All remaining configs are now lazy loaded - see _lazy_imports_registry.py - ### ADAPTERS ### - import litellm.anthropic_interface as anthropic +# Import LlmProviders here (before main import) because it's imported during import time +# in multiple places including openai.py (via main import) +from litellm.types.utils import LlmProviders - ### Vector Store Registry ### +## Lazy loading this is not straightforward, will leave it here for now. +from .main import * +from .compression import compress - ### RAG ### - from . import rag +# Skills API +from .skills.main import ( + create_skill, + acreate_skill, + list_skills, + alist_skills, + get_skill, + aget_skill, + delete_skill, + adelete_skill, +) +from .evals.main import ( + create_eval, + acreate_eval, + list_evals, + alist_evals, + get_eval, + aget_eval, + delete_eval, + adelete_eval, + cancel_eval, + acancel_eval, + create_run, + acreate_run, + list_runs, + alist_runs, + get_run, + aget_run, + delete_run, + adelete_run, + cancel_run, + acancel_run, +) +from .integrations import * +from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients +from .exceptions import ( + AuthenticationError, + InvalidRequestError, + BadRequestError, + ImageFetchError, + NotFoundError, + PermissionDeniedError, + RateLimitError, + RateLimitErrorCategory, + RateLimitType, + ServiceUnavailableError, + BadGatewayError, + OpenAIError, + ContextWindowExceededError, + ContentPolicyViolationError, + BudgetExceededError, + APIError, + Timeout, + APIConnectionError, + UnsupportedParamsError, + APIResponseValidationError, + UnprocessableEntityError, + InternalServerError, + JSONSchemaValidationError, + LITELLM_EXCEPTION_TYPES, + MockException, +) +from .budget_manager import BudgetManager +from .proxy.proxy_cli import run_server +from .router import Router +from .assistants.main import * +from .batches.main import * +from .images.main import * +from .videos.main import * +from .batch_completion.main import * +from .rerank_api.main import * +from .llms.anthropic.experimental_pass_through.messages.handler import * +from .responses.main import * - ### CUSTOM LLMs ### - - ### CLI UTILITIES ### - from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key - - ### PASSTHROUGH ### - from .passthrough import allm_passthrough_route, llm_passthrough_route - from .google_genai import agenerate_content +# Interactions API is available as litellm.interactions module +# Usage: litellm.interactions.create(), litellm.interactions.get(), etc. +from . import interactions +from .interactions.agents.main import ( + acreate as acreate_agent, + create as create_agent, + alist as alist_agents, + list as list_agents, + aget as aget_agent, + get as get_agent, + adelete as adelete_agent, + delete as delete_agent, + alist_versions as alist_agent_versions, + list_versions as list_agent_versions, +) +from .skills.main import ( + create_skill, + acreate_skill, + list_skills, + alist_skills, + get_skill, + aget_skill, + delete_skill, + adelete_skill, +) +from .containers.main import * +from .ocr.main import * +from .rust_bridge import rust +from .rag.main import * +from .sandbox.main import * +from .search.main import * +from .realtime_api.main import ( + _arealtime, + acreate_realtime_client_secret, + acreate_realtime_transcription_session, + arealtime_calls, +) +from .responses.main import _aresponses_websocket +from .fine_tuning.main import * +from .files.main import * +from .vector_store_files.main import ( + acreate as avector_store_file_create, + adelete as avector_store_file_delete, + alist as avector_store_file_list, + aretrieve as avector_store_file_retrieve, + aretrieve_content as avector_store_file_content, + aupdate as avector_store_file_update, + create as vector_store_file_create, + delete as vector_store_file_delete, + list as vector_store_file_list, + retrieve as vector_store_file_retrieve, + retrieve_content as vector_store_file_content, + update as vector_store_file_update, +) +from .scheduler import * ### ADAPTERS ### +from .types.adapter import AdapterItem +import litellm.anthropic_interface as anthropic + adapters: List[AdapterItem] = [] ### Vector Store Registry ### +from .vector_stores.vector_store_registry import ( + VectorStoreRegistry, + VectorStoreIndexRegistry, +) + vector_store_registry: Optional[VectorStoreRegistry] = None vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None +### RAG ### +from . import rag + ### CUSTOM LLMs ### +from .types.llms.custom_llm import CustomLLMItem + custom_provider_map: List[CustomLLMItem] = [] _custom_providers: List[str] = [] # internal helper util, used to track names of custom providers disable_hf_tokenizer_download: Optional[bool] = ( @@ -1471,6 +1478,13 @@ disable_hf_tokenizer_download: Optional[bool] = ( ) global_disable_no_log_param: bool = False +### CLI UTILITIES ### +from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key + +### PASSTHROUGH ### +from .passthrough import allm_passthrough_route, llm_passthrough_route +from .google_genai import agenerate_content + ### GLOBAL CONFIG ### global_bitbucket_config: Optional[Dict[str, Any]] = None @@ -1494,21 +1508,10 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: # Lazy loading system for heavy modules to reduce initial import time and memory usage if TYPE_CHECKING: - import httpx - from litellm.types.utils import ModelInfo as _ModelInfoType from litellm.types.utils import PriorityReservationSettings from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.caching.caching import Cache - from litellm.types.adapter import AdapterItem - from litellm.types.integrations.datadog import DatadogInitParams - from litellm.types.integrations.newrelic import NewRelicInitParams - from litellm.types.llms.custom_llm import CustomLLMItem - from litellm.types.prompts.init_prompts import PromptSpec - from litellm.vector_stores.vector_store_registry import ( - VectorStoreIndexRegistry, - VectorStoreRegistry, - ) # Type stubs for lazy-loaded configs to help mypy from .llms.bedrock.chat.converse_transformation import ( @@ -2184,6 +2187,16 @@ if TYPE_CHECKING: # Track if async client cleanup has been registered (for lazy loading) _async_client_cleanup_registered = False +# Eager loading for backwards compatibility with VCR and other HTTP recording tools +# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time +# For now, this only affects encoding (tiktoken) as it was the only reported issue +# See: https://github.com/BerriAI/litellm/issues/18659 +# This ensures encoding is initialized before VCR starts recording HTTP requests +if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"): + # Load encoding at import time (pre-#18070 behavior) + # This ensures encoding is initialized before VCR starts recording + from .main import encoding + def __getattr__(name: str) -> Any: """Lazy import handler with cached registry for improved performance.""" @@ -2263,8 +2276,6 @@ def __getattr__(name: str) -> Any: "openAIGPT5Config": "OpenAIGPT5Config", "nvidiaNimConfig": "NvidiaNimConfig", "nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig", - "vertexAITextEmbeddingConfig": "VertexAITextEmbeddingConfig", - "_key_management_settings": "KeyManagementSettings", } if name in _config_instances: from ._lazy_imports import get_litellm_globals @@ -2382,30 +2393,7 @@ def __getattr__(name: str) -> Any: return locals()[name] - from ._lazy_imports import lazy_import_litellm_submodule - - submodule: Final = lazy_import_litellm_submodule(name) - if submodule is not None: - return submodule - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -from ._lazy_imports import LiteLLMModule -from ._lazy_imports_registry import STAR_IMPORT_PUBLIC_NAMES - -sys.modules[__name__].__class__ = LiteLLMModule - -__all__ = list(STAR_IMPORT_PUBLIC_NAMES) # mutable-ok: star imports require __all__ to be a list of str - - # ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time - -# Eager loading for backwards compatibility with VCR and other HTTP recording tools -# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time -# For now, this only affects encoding (tiktoken) as it was the only reported issue -# See: https://github.com/BerriAI/litellm/issues/18659 -# This ensures encoding is initialized before VCR starts recording HTTP requests -# This block stays at the bottom so __getattr__ can resolve attributes main.py needs during its import -if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"): - from .main import encoding diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 004297a559e..fe3c7c264ee 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -16,10 +16,9 @@ until they're actually needed. """ import importlib -import importlib.util import sys from collections.abc import Callable, Mapping -from types import MappingProxyType, ModuleType +from types import ModuleType from typing import TYPE_CHECKING, Any, Final, cast from typing_extensions import ReadOnly, TypedDict @@ -35,8 +34,6 @@ from ._lazy_imports_registry import ( _LITELLM_LOGGING_IMPORT_MAP, _LLM_CONFIGS_IMPORT_MAP, _LLM_PROVIDER_LOGIC_IMPORT_MAP, - _SDK_MODULE_ALIASES, - _SDK_SYMBOLS_IMPORT_MAP, _TOKEN_COUNTER_IMPORT_MAP, _TYPES_IMPORT_MAP, _TYPES_UTILS_IMPORT_MAP, @@ -81,10 +78,7 @@ def _get_utils_globals() -> dict[str, object]: This is where we cache imported attributes so we don't import them twice. When you do `litellm.utils.some_function`, it gets stored in this dictionary. """ - cached: Final = sys.modules.get("litellm.utils") - if cached is not None: - return cached.__dict__ - return importlib.import_module("litellm.utils").__dict__ + return sys.modules["litellm.utils"].__dict__ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None": @@ -220,10 +214,6 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], object]]: _LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic for name in UTILS_MODULE_NAMES: _LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils_module - for name in _SDK_SYMBOLS_IMPORT_MAP: - _LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_symbols) - for name in _SDK_MODULE_ALIASES: - _LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_module_alias) return _LAZY_IMPORT_REGISTRY @@ -360,86 +350,6 @@ def _lazy_import_llm_provider_logic(name: str) -> object: return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic") -def _lazy_import_sdk_symbols(name: str) -> object: - """Handler for SDK symbols previously imported eagerly at the bottom of litellm/__init__.py""" - return _generic_lazy_import(name, _SDK_SYMBOLS_IMPORT_MAP, "SDK symbols") - - -def _lazy_import_sdk_module_alias(name: str) -> object: - """Handler for litellm attributes that bind a module (e.g. litellm.anthropic)""" - _globals: Final = get_litellm_globals() - if name in _globals: - return _globals[name] - module: Final = importlib.import_module(_SDK_MODULE_ALIASES[name]) - _globals[name] = module # rebind-ok: caches the resolved module alias on the package - return module - - -_SHADOWABLE_SDK_FUNCTIONS: Final = MappingProxyType( - { - "batch_completion": ("litellm.batch_completion.main", "batch_completion"), - "ocr": ("litellm.ocr.main", "ocr"), - "responses": ("litellm.responses.main", "responses"), - "search": ("litellm.search.main", "search"), - } -) - - -def _shadowable_function_property(name: str) -> property: - """Property keeping litellm. bound to the SDK function even after the import - machinery binds the identically named litellm. subpackage onto the litellm module.""" - module_path, attr_name = _SHADOWABLE_SDK_FUNCTIONS[name] - - def _get(module: ModuleType) -> object: - stored: Final = module.__dict__.get(name) - if stored is not None and not (isinstance(stored, ModuleType) and stored.__name__ == f"litellm.{name}"): - return stored - value: Final = _module_attribute(importlib.import_module(module_path), attr_name) - module.__dict__[name] = value # rebind-ok: caches the resolved function on the litellm module - return value - - def _set(module: ModuleType, value: object) -> None: - module.__dict__[name] = value # rebind-ok: property setter must store assignments on the module - - return property(_get, _set) - - -class LiteLLMModule(ModuleType): - """Module type installed on the litellm package so function names shadowed by - same-named subpackages (litellm.responses, ...) keep resolving to the functions.""" - - batch_completion = _shadowable_function_property("batch_completion") - ocr = _shadowable_function_property("ocr") - responses = _shadowable_function_property("responses") - search = _shadowable_function_property("search") - - -def lazy_import_submodule(package: str, name: str) -> "ModuleType | None": - """Resolve . as a submodule (e.g. litellm.utils) when no other handler matches""" - if name.startswith("__") or not name.isidentifier(): - return None - qualified_name: Final = f"{package}.{name}" - try: - spec: Final = importlib.util.find_spec(qualified_name) - except ModuleNotFoundError: - return None - if spec is None: - return None - try: - module: Final = importlib.import_module(qualified_name) - except ModuleNotFoundError as exc: - if exc.name == qualified_name: - return None - raise - sys.modules[package].__dict__[name] = module # rebind-ok: caches the resolved submodule on the package - return module - - -def lazy_import_litellm_submodule(name: str) -> "ModuleType | None": - """Resolve litellm. as a submodule (e.g. litellm.utils) when no other handler matches""" - return lazy_import_submodule("litellm", name) - - def _lazy_import_utils_module(name: str) -> object: """ Handler for utils module lazy imports. diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index b0e2fb1398c..e9199e1ec80 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -5,8 +5,6 @@ This module contains all the name tuples and import maps used by the lazy import Separated from the handler functions for better organization. """ -from collections.abc import Mapping -from types import MappingProxyType from typing import Final # Cost calculator names that support lazy loading via _lazy_import_cost_calculator @@ -1481,1171 +1479,6 @@ _UTILS_MODULE_IMPORT_MAP: Final = { "LiteLLM_Params": ("litellm.types.router", "LiteLLM_Params"), } -_SDK_SYMBOLS_IMPORT_MAP: Final[Mapping[str, tuple[str, str]]] = MappingProxyType( - { - "AI21Config": ("litellm.llms.ai21.chat.transformation", "AI21ChatConfig"), - "ALL_RESPONSES_API_TOOL_PARAMS": ("litellm.assistants.main", "ALL_RESPONSES_API_TOOL_PARAMS"), - "APIConnectionError": ("litellm.exceptions", "APIConnectionError"), - "APIError": ("litellm.exceptions", "APIError"), - "APIResponseValidationError": ("litellm.exceptions", "APIResponseValidationError"), - "AZURE_OPENAI_AUDIO_PROVIDERS": ("litellm.main", "AZURE_OPENAI_AUDIO_PROVIDERS"), - "AdapterCompletionStreamWrapper": ("litellm.types.utils", "AdapterCompletionStreamWrapper"), - "AdapterItem": ("litellm.types.adapter", "AdapterItem"), - "AdaptiveRouterConfig": ("litellm.types.router", "AdaptiveRouterConfig"), - "AdaptiveRouterPreferences": ("litellm.types.router", "AdaptiveRouterPreferences"), - "AdaptiveRouterWeights": ("litellm.types.router", "AdaptiveRouterWeights"), - "AlephAlphaConfig": ("litellm.llms.deprecated_providers.aleph_alpha", "AlephAlphaConfig"), - "AlertingConfig": ("litellm.types.router", "AlertingConfig"), - "AllEmbeddingInputValues": ("litellm.assistants.main", "AllEmbeddingInputValues"), - "AllMessageValues": ("litellm.assistants.main", "AllMessageValues"), - "AllPromptValues": ("litellm.assistants.main", "AllPromptValues"), - "AllowedFailsPolicy": ("litellm.types.router", "AllowedFailsPolicy"), - "AmazonTitanV2Config": ("litellm.llms.bedrock.embed.amazon_titan_v2_transformation", "AmazonTitanV2Config"), - "Annotated": ("litellm.assistants.main", "Annotated"), - "AnthropicBatchesHandler": ("litellm.llms.anthropic.batches.handler", "AnthropicBatchesHandler"), - "AnthropicChatCompletion": ("litellm.llms.anthropic.chat.handler", "AnthropicChatCompletion"), - "AnthropicMessagesRequestUtils": ( - "litellm.llms.anthropic.experimental_pass_through.messages.utils", - "AnthropicMessagesRequestUtils", - ), - "AnthropicMessagesResponse": ( - "litellm.types.llms.anthropic_messages.anthropic_response", - "AnthropicMessagesResponse", - ), - "AnthropicMetadata": ("litellm.types.llms.anthropic_messages.anthropic_request", "AnthropicMetadata"), - "AnthropicModelInfo": ("litellm.llms.anthropic.common_utils", "AnthropicModelInfo"), - "Assistant": ("litellm.assistants.main", "Assistant"), - "AssistantDeleted": ("litellm.assistants.main", "AssistantDeleted"), - "AssistantEventHandler": ("litellm.assistants.main", "AssistantEventHandler"), - "AssistantStreamManager": ("litellm.assistants.main", "AssistantStreamManager"), - "AssistantToolParam": ("litellm.assistants.main", "AssistantToolParam"), - "AssistantsTypedDict": ("litellm.types.router", "AssistantsTypedDict"), - "AsyncAssistantEventHandler": ("litellm.assistants.main", "AsyncAssistantEventHandler"), - "AsyncAssistantStreamManager": ("litellm.assistants.main", "AsyncAssistantStreamManager"), - "AsyncCompletions": ("litellm.main", "AsyncCompletions"), - "AsyncCursorPage": ("litellm.assistants.main", "AsyncCursorPage"), - "AsyncIterator": ("litellm.llms.anthropic.experimental_pass_through.messages.handler", "AsyncIterator"), - "AsyncOpenAI": ("litellm.assistants.main", "AsyncOpenAI"), - "Attachment": ("litellm.types.llms.openai", "Attachment"), - "AttachmentTool": ("litellm.assistants.main", "AttachmentTool"), - "AuthenticationError": ("litellm.exceptions", "AuthenticationError"), - "AutoRouterCapabilityLimit": ("litellm.types.router", "AutoRouterCapabilityLimit"), - "AzureAIEmbedding": ("litellm.llms.azure_ai.embed.handler", "AzureAIEmbedding"), - "AzureAnthropicChatCompletion": ("litellm.llms.azure_ai.anthropic.handler", "AzureAnthropicChatCompletion"), - "AzureAssistantsAPI": ("litellm.llms.azure.assistants", "AzureAssistantsAPI"), - "AzureAudioTranscription": ("litellm.llms.azure.audio_transcriptions", "AzureAudioTranscription"), - "AzureBatchesAPI": ("litellm.llms.azure.batches.handler", "AzureBatchesAPI"), - "AzureChatCompletion": ("litellm.llms.azure.azure", "AzureChatCompletion"), - "AzureOpenAIFilesAPI": ("litellm.llms.azure.files.handler", "AzureOpenAIFilesAPI"), - "AzureOpenAIFineTuningAPI": ("litellm.llms.azure.fine_tuning.handler", "AzureOpenAIFineTuningAPI"), - "AzureOpenAIO1ChatCompletion": ("litellm.llms.azure.chat.o_series_handler", "AzureOpenAIO1ChatCompletion"), - "AzureTextCompletion": ("litellm.llms.azure.completion.handler", "AzureTextCompletion"), - "BATCH_GUARDRAIL_RESPONSE_FIELD": ("litellm.assistants.main", "BATCH_GUARDRAIL_RESPONSE_FIELD"), - "BadGatewayError": ("litellm.exceptions", "BadGatewayError"), - "BadRequestError": ("litellm.exceptions", "BadRequestError"), - "BaseConfig": ("litellm.llms.base_llm.chat.transformation", "BaseConfig"), - "BaseLLMAIOHTTPHandler": ("litellm.llms.custom_httpx.aiohttp_handler", "BaseLLMAIOHTTPHandler"), - "BaseLLMException": ("litellm.llms.base_llm.chat.transformation", "BaseLLMException"), - "BaseLLMHTTPHandler": ("litellm.llms.custom_httpx.llm_http_handler", "BaseLLMHTTPHandler"), - "BaseLiteLLMOpenAIResponseObject": ("litellm.types.llms.base", "BaseLiteLLMOpenAIResponseObject"), - "BaseModel": ("litellm.scheduler", "BaseModel"), - "BaseResponsesAPIConfig": ("litellm.llms.base_llm.responses.transformation", "BaseResponsesAPIConfig"), - "BaseResponsesAPIStreamingIterator": ( - "litellm.responses.streaming_iterator", - "BaseResponsesAPIStreamingIterator", - ), - "Batch": ("litellm.assistants.main", "Batch"), - "BatchGuardrailRecord": ("litellm.types.llms.openai", "BatchGuardrailRecord"), - "BatchGuardrailReport": ("litellm.types.llms.openai", "BatchGuardrailReport"), - "BatchJobStatus": ("litellm.assistants.main", "BatchJobStatus"), - "BatchRequestCounts": ("litellm.batches.main", "BatchRequestCounts"), - "BedrockBatchesHandler": ("litellm.llms.bedrock.batches.handler", "BedrockBatchesHandler"), - "BedrockConverseLLM": ("litellm.llms.bedrock.chat.converse_handler", "BedrockConverseLLM"), - "BedrockEmbedding": ("litellm.llms.bedrock.embed.embedding", "BedrockEmbedding"), - "BedrockFilesHandler": ("litellm.llms.bedrock.files.handler", "BedrockFilesHandler"), - "BedrockImageEdit": ("litellm.llms.bedrock.image_edit.handler", "BedrockImageEdit"), - "BedrockImageGeneration": ("litellm.llms.bedrock.image_generation.image_handler", "BedrockImageGeneration"), - "BedrockRerankHandler": ("litellm.llms.bedrock.rerank.handler", "BedrockRerankHandler"), - "BudgetExceededError": ("litellm.exceptions", "BudgetExceededError"), - "BudgetManager": ("litellm.budget_manager", "BudgetManager"), - "CARRY_UNMATCHED_MESSAGE_POINTS": ("litellm.responses.main", "CARRY_UNMATCHED_MESSAGE_POINTS"), - "CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS": ("litellm.files.main", "CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS"), - "CREATE_FILE_REQUESTS_PURPOSE": ("litellm.assistants.main", "CREATE_FILE_REQUESTS_PURPOSE"), - "CallTypes": ("litellm.types.utils", "CallTypes"), - "CancelBatchRequest": ("litellm.types.llms.openai", "CancelBatchRequest"), - "CharacterObject": ("litellm.types.videos.main", "CharacterObject"), - "Chat": ("litellm.main", "Chat"), - "ChatCompletionAnnotation": ("litellm.types.llms.openai", "ChatCompletionAnnotation"), - "ChatCompletionAnnotationURLCitation": ("litellm.types.llms.openai", "ChatCompletionAnnotationURLCitation"), - "ChatCompletionAssistantContentValue": ("litellm.assistants.main", "ChatCompletionAssistantContentValue"), - "ChatCompletionAssistantMessage": ("litellm.types.llms.openai", "ChatCompletionAssistantMessage"), - "ChatCompletionAssistantToolCall": ("litellm.types.llms.openai", "ChatCompletionAssistantToolCall"), - "ChatCompletionAudioDelta": ("litellm.types.llms.openai", "ChatCompletionAudioDelta"), - "ChatCompletionAudioObject": ("litellm.types.llms.openai", "ChatCompletionAudioObject"), - "ChatCompletionAudioParam": ("litellm.assistants.main", "ChatCompletionAudioParam"), - "ChatCompletionCachedContent": ("litellm.types.llms.openai", "ChatCompletionCachedContent"), - "ChatCompletionChunk": ("litellm.assistants.main", "ChatCompletionChunk"), - "ChatCompletionContentPartInputAudioParam": ( - "litellm.assistants.main", - "ChatCompletionContentPartInputAudioParam", - ), - "ChatCompletionDeltaChunk": ("litellm.types.llms.openai", "ChatCompletionDeltaChunk"), - "ChatCompletionDeveloperMessage": ("litellm.types.llms.openai", "ChatCompletionDeveloperMessage"), - "ChatCompletionDocumentObject": ("litellm.types.llms.openai", "ChatCompletionDocumentObject"), - "ChatCompletionFileObject": ("litellm.types.llms.openai", "ChatCompletionFileObject"), - "ChatCompletionFileObjectFile": ("litellm.types.llms.openai", "ChatCompletionFileObjectFile"), - "ChatCompletionFunctionMessage": ("litellm.types.llms.openai", "ChatCompletionFunctionMessage"), - "ChatCompletionImageObject": ("litellm.types.llms.openai", "ChatCompletionImageObject"), - "ChatCompletionImageUrlObject": ("litellm.types.llms.openai", "ChatCompletionImageUrlObject"), - "ChatCompletionMessageToolCall": ("litellm.types.utils", "ChatCompletionMessageToolCall"), - "ChatCompletionModality": ("litellm.assistants.main", "ChatCompletionModality"), - "ChatCompletionNamedToolChoiceParam": ("litellm.types.llms.openai", "ChatCompletionNamedToolChoiceParam"), - "ChatCompletionPredictionContentParam": ("litellm.assistants.main", "ChatCompletionPredictionContentParam"), - "ChatCompletionReasoningItem": ("litellm.types.llms.openai", "ChatCompletionReasoningItem"), - "ChatCompletionReasoningSummaryTextBlock": ( - "litellm.types.llms.openai", - "ChatCompletionReasoningSummaryTextBlock", - ), - "ChatCompletionRedactedThinkingBlock": ("litellm.types.llms.openai", "ChatCompletionRedactedThinkingBlock"), - "ChatCompletionRequest": ("litellm.types.llms.openai", "ChatCompletionRequest"), - "ChatCompletionResponseMessage": ("litellm.types.llms.openai", "ChatCompletionResponseMessage"), - "ChatCompletionSystemMessage": ("litellm.types.llms.openai", "ChatCompletionSystemMessage"), - "ChatCompletionTextObject": ("litellm.types.llms.openai", "ChatCompletionTextObject"), - "ChatCompletionThinkingBlock": ("litellm.types.llms.openai", "ChatCompletionThinkingBlock"), - "ChatCompletionToolChoiceFunctionParam": ("litellm.types.llms.openai", "ChatCompletionToolChoiceFunctionParam"), - "ChatCompletionToolChoiceObjectParam": ("litellm.types.llms.openai", "ChatCompletionToolChoiceObjectParam"), - "ChatCompletionToolChoiceStringValues": ("litellm.assistants.main", "ChatCompletionToolChoiceStringValues"), - "ChatCompletionToolChoiceValues": ("litellm.assistants.main", "ChatCompletionToolChoiceValues"), - "ChatCompletionToolMessage": ("litellm.types.llms.openai", "ChatCompletionToolMessage"), - "ChatCompletionToolParam": ("litellm.types.llms.openai", "ChatCompletionToolParam"), - "ChatCompletionToolParamFunctionChunk": ("litellm.types.llms.openai", "ChatCompletionToolParamFunctionChunk"), - "ChatCompletionToolReferenceObject": ("litellm.types.llms.openai", "ChatCompletionToolReferenceObject"), - "ChatCompletionUsageBlock": ("litellm.types.llms.openai", "ChatCompletionUsageBlock"), - "ChatCompletionUserMessage": ("litellm.types.llms.openai", "ChatCompletionUserMessage"), - "ChatCompletionVideoObject": ("litellm.types.llms.openai", "ChatCompletionVideoObject"), - "ChatCompletionVideoUrlObject": ("litellm.types.llms.openai", "ChatCompletionVideoUrlObject"), - "Choices": ("litellm.types.utils", "Choices"), - "ChunkProcessor": ("litellm.litellm_core_utils.streaming_chunk_builder_utils", "ChunkProcessor"), - "CitationsObject": ("litellm.types.llms.openai", "CitationsObject"), - "ClassVar": ("litellm.files.main", "ClassVar"), - "ClassifierPlugin": ("litellm.types.router", "ClassifierPlugin"), - "CodeInterpreterToolParam": ("litellm.types.llms.openai", "CodeInterpreterToolParam"), - "CodestralTextCompletion": ("litellm.llms.codestral.completion.handler", "CodestralTextCompletion"), - "CompletionRequest": ("litellm.types.completion", "CompletionRequest"), - "CompletionTimeout": ("litellm.litellm_core_utils.completion_timeout", "CompletionTimeout"), - "CompletionTokensDetails": ("litellm.main", "CompletionTokensDetails"), - "Completions": ("litellm.main", "Completions"), - "ComputerToolParam": ("litellm.types.llms.openai", "ComputerToolParam"), - "ConfigDict": ("litellm.files.main", "ConfigDict"), - "ConfigurableClientsideParamsCustomAuth": ("litellm.types.router", "ConfigurableClientsideParamsCustomAuth"), - "ConsumedRequestTagsStamp": ("litellm.types.router", "ConsumedRequestTagsStamp"), - "ContentPartAddedEvent": ("litellm.types.llms.openai", "ContentPartAddedEvent"), - "ContentPartDoneEvent": ("litellm.types.llms.openai", "ContentPartDoneEvent"), - "ContentPartDonePartOutputText": ("litellm.types.llms.openai", "ContentPartDonePartOutputText"), - "ContentPartDonePartReasoningText": ("litellm.types.llms.openai", "ContentPartDonePartReasoningText"), - "ContentPartDonePartRefusal": ("litellm.types.llms.openai", "ContentPartDonePartRefusal"), - "ContentPolicyViolationError": ("litellm.exceptions", "ContentPolicyViolationError"), - "ContextManagementEntry": ("litellm.types.llms.openai", "ContextManagementEntry"), - "ContextWindowExceededError": ("litellm.exceptions", "ContextWindowExceededError"), - "Coroutine": ("litellm.files.main", "Coroutine"), - "CreateBatchRequest": ("litellm.types.llms.openai", "CreateBatchRequest"), - "CreateFileRequest": ("litellm.types.llms.openai", "CreateFileRequest"), - "CreateVideoRequest": ("litellm.types.llms.openai", "CreateVideoRequest"), - "CredentialLiteLLMParams": ("litellm.types.router", "CredentialLiteLLMParams"), - "CustomLLM": ("litellm.llms.custom_llm", "CustomLLM"), - "CustomLLMItem": ("litellm.types.llms.custom_llm", "CustomLLMItem"), - "CustomPricingLiteLLMParams": ("litellm.types.utils", "CustomPricingLiteLLMParams"), - "CustomRoutingStrategyBase": ("litellm.types.router", "CustomRoutingStrategyBase"), - "CustomToolCallOutputItem": ("litellm.types.responses.main", "CustomToolCallOutputItem"), - "DEFAULT_IMAGE_ENDPOINT_MODEL": ("litellm.images.main", "DEFAULT_IMAGE_ENDPOINT_MODEL"), - "DEFAULT_IN_MEMORY_TTL": ("litellm.scheduler", "DEFAULT_IN_MEMORY_TTL"), - "DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT": ( - "litellm.main", - "DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", - ), - "DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT": ("litellm.main", "DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT"), - "DEFAULT_POLLING_INTERVAL": ("litellm.scheduler", "DEFAULT_POLLING_INTERVAL"), - "DEFAULT_REQUEST_TIMEOUT": ("litellm.videos.main", "DEFAULT_REQUEST_TIMEOUT"), - "DEFAULT_VIDEO_ENDPOINT_MODEL": ("litellm.videos.main", "DEFAULT_VIDEO_ENDPOINT_MODEL"), - "DatabricksEmbeddingHandler": ("litellm.llms.databricks.embed.handler", "DatabricksEmbeddingHandler"), - "DatadogInitParams": ("litellm.types.integrations.datadog", "DatadogInitParams"), - "DecodedResponseId": ("litellm.types.responses.main", "DecodedResponseId"), - "DeleteResponseResult": ("litellm.types.responses.main", "DeleteResponseResult"), - "Deployment": ("litellm.types.router", "Deployment"), - "DeploymentTypedDict": ("litellm.types.router", "DeploymentTypedDict"), - "Discriminator": ("litellm.assistants.main", "Discriminator"), - "DocumentObject": ("litellm.types.llms.openai", "DocumentObject"), - "EmbeddingCreateParams": ("litellm.assistants.main", "EmbeddingCreateParams"), - "EmbeddingInput": ("litellm.assistants.main", "EmbeddingInput"), - "EmbeddingRequest": ("litellm.types.embedding", "EmbeddingRequest"), - "Enum": ("litellm.assistants.main", "Enum"), - "ErrorEvent": ("litellm.types.llms.openai", "ErrorEvent"), - "ErrorEventError": ("litellm.types.llms.openai", "ErrorEventError"), - "FIRST_COMPLETED": ("litellm.batch_completion.main", "FIRST_COMPLETED"), - "FORWARDED_KWARGS_KEYS": ("litellm.main", "FORWARDED_KWARGS_KEYS"), - "FallbackAccessCheck": ("litellm.types.router", "FallbackAccessCheck"), - "Field": ("litellm.files.main", "Field"), - "FileContent": ("litellm.videos.main", "FileContent"), - "FileContentProvider": ("litellm.files.main", "FileContentProvider"), - "FileContentRequest": ("litellm.types.llms.openai", "FileContentRequest"), - "FileContentStreamingResponse": ("litellm.files.streaming", "FileContentStreamingResponse"), - "FileContentStreamingResult": ("litellm.files.types", "FileContentStreamingResult"), - "FileCreateProvider": ("litellm.files.main", "FileCreateProvider"), - "FileDeleteProvider": ("litellm.files.main", "FileDeleteProvider"), - "FileDeleted": ("litellm.files.main", "FileDeleted"), - "FileExpiresAfter": ("litellm.types.llms.openai", "FileExpiresAfter"), - "FileListPage": ("litellm.types.llms.openai", "FileListPage"), - "FileListProvider": ("litellm.files.main", "FileListProvider"), - "FileObject": ("litellm.files.main", "FileObject"), - "FileRetrieveProvider": ("litellm.files.main", "FileRetrieveProvider"), - "FileSearchCallCompletedEvent": ("litellm.types.llms.openai", "FileSearchCallCompletedEvent"), - "FileSearchCallInProgressEvent": ("litellm.types.llms.openai", "FileSearchCallInProgressEvent"), - "FileSearchCallSearchingEvent": ("litellm.types.llms.openai", "FileSearchCallSearchingEvent"), - "FileSearchTool": ("litellm.types.llms.openai", "FileSearchTool"), - "FileSearchToolParam": ("litellm.types.llms.openai", "FileSearchToolParam"), - "FileTypes": ("litellm.files.main", "FileTypes"), - "FineTuningConfig": ("litellm.types.router", "FineTuningConfig"), - "FineTuningJob": ("litellm.assistants.main", "FineTuningJob"), - "FineTuningJobCreate": ("litellm.types.llms.openai", "FineTuningJobCreate"), - "FlowItem": ("litellm.scheduler", "FlowItem"), - "Function": ("litellm.types.llms.openai", "Function"), - "FunctionCallArgumentsDeltaEvent": ("litellm.types.llms.openai", "FunctionCallArgumentsDeltaEvent"), - "FunctionCallArgumentsDoneEvent": ("litellm.types.llms.openai", "FunctionCallArgumentsDoneEvent"), - "GeminiModelInfo": ("litellm.llms.gemini.common_utils", "GeminiModelInfo"), - "GenAIHubOrchestration": ("litellm.llms.sap.chat.handler", "GenAIHubOrchestration"), - "Generator": ("litellm.responses.main", "Generator"), - "Generic": ("litellm.files.main", "Generic"), - "GenericBudgetWindowDetails": ("litellm.types.router", "GenericBudgetWindowDetails"), - "GenericChatCompletionMessage": ("litellm.types.llms.openai", "GenericChatCompletionMessage"), - "GenericEvent": ("litellm.types.llms.openai", "GenericEvent"), - "GenericLiteLLMParams": ("litellm.types.router", "GenericLiteLLMParams"), - "GenericResponseOutputItem": ("litellm.types.responses.main", "GenericResponseOutputItem"), - "GenericResponseOutputItemContentAnnotation": ( - "litellm.types.responses.main", - "GenericResponseOutputItemContentAnnotation", - ), - "GoogleBatchEmbeddings": ( - "litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler", - "GoogleBatchEmbeddings", - ), - "GroqChatCompletion": ("litellm.llms.groq.chat.handler", "GroqChatCompletion"), - "GuardrailLiteLLMParams": ("litellm.types.router", "GuardrailLiteLLMParams"), - "GuardrailTypedDict": ("litellm.types.router", "GuardrailTypedDict"), - "HiddenParams": ("litellm.types.llms.base", "HiddenParams"), - "HttpxBinaryResponseContent": ("litellm.types.llms.openai", "HttpxBinaryResponseContent"), - "HuggingFaceEmbedding": ("litellm.llms.huggingface.embedding.handler", "HuggingFaceEmbedding"), - "Hyperparameters": ("litellm.types.llms.openai", "Hyperparameters"), - "IBMWatsonXMixin": ("litellm.llms.watsonx.common_utils", "IBMWatsonXMixin"), - "IO": ("litellm.assistants.main", "IO"), - "IOBase": ("litellm.ocr.main", "IOBase"), - "ImageEditOptionalRequestParams": ("litellm.types.images.main", "ImageEditOptionalRequestParams"), - "ImageFetchError": ("litellm.exceptions", "ImageFetchError"), - "ImageFileObject": ("litellm.types.llms.openai", "ImageFileObject"), - "ImageGenerationPartialImageEvent": ("litellm.types.llms.openai", "ImageGenerationPartialImageEvent"), - "ImageGenerationRequestQuality": ("litellm.types.llms.openai", "ImageGenerationRequestQuality"), - "ImageURLListItem": ("litellm.types.llms.openai", "ImageURLListItem"), - "ImageURLObject": ("litellm.types.llms.openai", "ImageURLObject"), - "IncompleteDetails": ("litellm.assistants.main", "IncompleteDetails"), - "InputTokensDetails": ("litellm.types.llms.openai", "InputTokensDetails"), - "InternalServerError": ("litellm.exceptions", "InternalServerError"), - "InvalidRequestError": ("litellm.exceptions", "InvalidRequestError"), - "Iterable": ("litellm.responses.main", "Iterable"), - "Iterator": ("litellm.llms.anthropic.experimental_pass_through.messages.handler", "Iterator"), - "JSONProviderRegistry": ("litellm.llms.openai_like.json_loader", "JSONProviderRegistry"), - "JSONSchemaValidationError": ("litellm.exceptions", "JSONSchemaValidationError"), - "KeyManagementSettings": ("litellm.types.secret_managers.main", "KeyManagementSettings"), - "LIST_BATCHES_SUPPORTED_PROVIDERS": ("litellm.batches.main", "LIST_BATCHES_SUPPORTED_PROVIDERS"), - "LITELLM_EXCEPTION_TYPES": ("litellm.exceptions", "LITELLM_EXCEPTION_TYPES"), - "LITELLM_IMAGE_VARIATION_PROVIDERS": ("litellm.types.utils", "LITELLM_IMAGE_VARIATION_PROVIDERS"), - "ListBatchRequest": ("litellm.types.llms.openai", "ListBatchRequest"), - "ListBatchesSupportedProvider": ("litellm.batches.main", "ListBatchesSupportedProvider"), - "LiteLLM": ("litellm.main", "LiteLLM"), - "LiteLLMBatch": ("litellm.types.utils", "LiteLLMBatch"), - "LiteLLMBatchCreateRequest": ("litellm.types.llms.openai", "LiteLLMBatchCreateRequest"), - "LiteLLMCompletionTransformationHandler": ( - "litellm.responses.litellm_completion_transformation.handler", - "LiteLLMCompletionTransformationHandler", - ), - "LiteLLMFineTuningJob": ("litellm.types.utils", "LiteLLMFineTuningJob"), - "LiteLLMFineTuningJobCreate": ("litellm.types.llms.openai", "LiteLLMFineTuningJobCreate"), - "LiteLLMLoggingObj": ("litellm.files.main", "LiteLLMLoggingObj"), - "LiteLLMMessagesToCompletionTransformationHandler": ( - "litellm.llms.anthropic.experimental_pass_through.adapters.handler", - "LiteLLMMessagesToCompletionTransformationHandler", - ), - "LiteLLMMessagesToResponsesAPIHandler": ( - "litellm.llms.anthropic.experimental_pass_through.responses_adapters.handler", - "LiteLLMMessagesToResponsesAPIHandler", - ), - "LiteLLMParamsTypedDict": ("litellm.types.router", "LiteLLMParamsTypedDict"), - "LiteLLMResponsesTransformationHandler": ( - "litellm.completion_extras.litellm_responses_transformation.transformation", - "LiteLLMResponsesTransformationHandler", - ), - "LiteLLMUnknownProvider": ("litellm.exceptions", "LiteLLMUnknownProvider"), - "LiteLLM_RouterFileObject": ("litellm.types.router", "LiteLLM_RouterFileObject"), - "LlmProviders": ("litellm.types.utils", "LlmProviders"), - "MCPCallArgumentsDeltaEvent": ("litellm.types.llms.openai", "MCPCallArgumentsDeltaEvent"), - "MCPCallArgumentsDoneEvent": ("litellm.types.llms.openai", "MCPCallArgumentsDoneEvent"), - "MCPCallCompletedEvent": ("litellm.types.llms.openai", "MCPCallCompletedEvent"), - "MCPCallFailedEvent": ("litellm.types.llms.openai", "MCPCallFailedEvent"), - "MCPCallInProgressEvent": ("litellm.types.llms.openai", "MCPCallInProgressEvent"), - "MCPListToolsCompletedEvent": ("litellm.types.llms.openai", "MCPListToolsCompletedEvent"), - "MCPListToolsFailedEvent": ("litellm.types.llms.openai", "MCPListToolsFailedEvent"), - "MCPListToolsInProgressEvent": ("litellm.types.llms.openai", "MCPListToolsInProgressEvent"), - "MCPTool": ("litellm.responses.main", "MCPTool"), - "MOCK_RESPONSE_TYPE": ("litellm.main", "MOCK_RESPONSE_TYPE"), - "Mapping": ("litellm.files.main", "Mapping"), - "MappingProxyType": ("litellm.main", "MappingProxyType"), - "Message": ("litellm.types.utils", "Message"), - "MessageContent": ("litellm.assistants.main", "MessageContent"), - "MessageContentImageFileObject": ("litellm.types.llms.openai", "MessageContentImageFileObject"), - "MessageContentImageURLObject": ("litellm.types.llms.openai", "MessageContentImageURLObject"), - "MessageContentTextObject": ("litellm.types.llms.openai", "MessageContentTextObject"), - "MessageData": ("litellm.types.llms.openai", "MessageData"), - "MirroredPricingParams": ("litellm.types.utils", "MirroredPricingParams"), - "MockException": ("litellm.exceptions", "MockException"), - "MockRouterTestingParams": ("litellm.types.router", "MockRouterTestingParams"), - "ModelConfig": ("litellm.types.router", "ModelConfig"), - "ModelGroupInfo": ("litellm.types.router", "ModelGroupInfo"), - "ModelGroupSettings": ("litellm.types.router", "ModelGroupSettings"), - "ModelInfo": ("litellm.types.router", "ModelInfo"), - "NOT_GIVEN": ("litellm.types.llms.openai", "NOT_GIVEN"), - "NewRelicInitParams": ("litellm.types.integrations.newrelic", "NewRelicInitParams"), - "NonNegativeInt": ("litellm.assistants.main", "NonNegativeInt"), - "NotFoundError": ("litellm.exceptions", "NotFoundError"), - "NotGiven": ("litellm.types.llms.openai", "NotGiven"), - "NotRequired": ("litellm.assistants.main", "NotRequired"), - "NvidiaRivaAudioTranscription": ( - "litellm.llms.nvidia_riva.audio_transcription.handler", - "NvidiaRivaAudioTranscription", - ), - "NvidiaRivaAudioTranscriptionConfig": ( - "litellm.llms.nvidia_riva.audio_transcription.transformation", - "NvidiaRivaAudioTranscriptionConfig", - ), - "OCRResponse": ("litellm.llms.base_llm.ocr.transformation", "OCRResponse"), - "OCR_REQUEST_FORMAT_PARAM": ("litellm.ocr.main", "OCR_REQUEST_FORMAT_PARAM"), - "OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS": ( - "litellm.files.main", - "OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS", - ), - "OPTIONAL_KWARGS_KEYS": ("litellm.main", "OPTIONAL_KWARGS_KEYS"), - "Omit": ("litellm.assistants.main", "Omit"), - "OpenAI": ("litellm.assistants.main", "OpenAI"), - "OpenAIAssistantsAPI": ("litellm.llms.openai.openai", "OpenAIAssistantsAPI"), - "OpenAIAudioTranscription": ("litellm.llms.openai.transcriptions.handler", "OpenAIAudioTranscription"), - "OpenAIAudioTranscriptionOptionalParams": ("litellm.assistants.main", "OpenAIAudioTranscriptionOptionalParams"), - "OpenAIBatchResponse": ("litellm.types.llms.openai", "OpenAIBatchResponse"), - "OpenAIBatchResult": ("litellm.types.llms.openai", "OpenAIBatchResult"), - "OpenAIBatchesAPI": ("litellm.llms.openai.openai", "OpenAIBatchesAPI"), - "OpenAIChatCompletion": ("litellm.llms.openai.openai", "OpenAIChatCompletion"), - "OpenAIChatCompletionAssistantMessage": ("litellm.types.llms.openai", "OpenAIChatCompletionAssistantMessage"), - "OpenAIChatCompletionChoices": ("litellm.types.llms.openai", "OpenAIChatCompletionChoices"), - "OpenAIChatCompletionChunk": ("litellm.types.llms.openai", "OpenAIChatCompletionChunk"), - "OpenAIChatCompletionDeveloperMessage": ("litellm.types.llms.openai", "OpenAIChatCompletionDeveloperMessage"), - "OpenAIChatCompletionFinishReason": ("litellm.assistants.main", "OpenAIChatCompletionFinishReason"), - "OpenAIChatCompletionLogprobs": ("litellm.types.llms.openai", "OpenAIChatCompletionLogprobs"), - "OpenAIChatCompletionLogprobsContent": ("litellm.types.llms.openai", "OpenAIChatCompletionLogprobsContent"), - "OpenAIChatCompletionLogprobsContentTopLogprobs": ( - "litellm.types.llms.openai", - "OpenAIChatCompletionLogprobsContentTopLogprobs", - ), - "OpenAIChatCompletionResponse": ("litellm.types.llms.openai", "OpenAIChatCompletionResponse"), - "OpenAIChatCompletionSystemMessage": ("litellm.types.llms.openai", "OpenAIChatCompletionSystemMessage"), - "OpenAIChatCompletionTextObject": ("litellm.types.llms.openai", "OpenAIChatCompletionTextObject"), - "OpenAIChatCompletionToolParam": ("litellm.types.llms.openai", "OpenAIChatCompletionToolParam"), - "OpenAIChatCompletionUserMessage": ("litellm.types.llms.openai", "OpenAIChatCompletionUserMessage"), - "OpenAICreateFileRequestOptionalParams": ("litellm.assistants.main", "OpenAICreateFileRequestOptionalParams"), - "OpenAICreateThreadParamsMessage": ("litellm.assistants.main", "OpenAICreateThreadParamsMessage"), - "OpenAICreateThreadParamsToolResources": ("litellm.types.llms.openai", "OpenAICreateThreadParamsToolResources"), - "OpenAIEmbedding": ("litellm.assistants.main", "OpenAIEmbedding"), - "OpenAIError": ("litellm.exceptions", "OpenAIError"), - "OpenAIErrorBody": ("litellm.types.llms.openai", "OpenAIErrorBody"), - "OpenAIFileObject": ("litellm.types.llms.openai", "OpenAIFileObject"), - "OpenAIFilesAPI": ("litellm.llms.openai.openai", "OpenAIFilesAPI"), - "OpenAIFilesPurpose": ("litellm.assistants.main", "OpenAIFilesPurpose"), - "OpenAIFineTuningAPI": ("litellm.llms.openai.fine_tuning.handler", "OpenAIFineTuningAPI"), - "OpenAIImageEditOptionalParams": ("litellm.assistants.main", "OpenAIImageEditOptionalParams"), - "OpenAIImageGenerationOptionalParams": ("litellm.assistants.main", "OpenAIImageGenerationOptionalParams"), - "OpenAIImageVariationOptionalParams": ("litellm.assistants.main", "OpenAIImageVariationOptionalParams"), - "OpenAIImageVariationsHandler": ( - "litellm.llms.openai.image_variations.handler", - "OpenAIImageVariationsHandler", - ), - "OpenAILikeChatHandler": ("litellm.llms.openai_like.chat.handler", "OpenAILikeChatHandler"), - "OpenAILikeEmbeddingHandler": ("litellm.llms.openai_like.embedding.handler", "OpenAILikeEmbeddingHandler"), - "OpenAILikeResponsesConfig": ( - "litellm.llms.openai_like.responses.transformation", - "OpenAILikeResponsesConfig", - ), - "OpenAIMcpServerTool": ("litellm.types.llms.openai", "OpenAIMcpServerTool"), - "OpenAIMessage": ("litellm.assistants.main", "OpenAIMessage"), - "OpenAIMessageContent": ("litellm.assistants.main", "OpenAIMessageContent"), - "OpenAIMessageContentListBlock": ("litellm.assistants.main", "OpenAIMessageContentListBlock"), - "OpenAIModerationResponse": ("litellm.types.llms.openai", "OpenAIModerationResponse"), - "OpenAIModerationResult": ("litellm.types.llms.openai", "OpenAIModerationResult"), - "OpenAIRealtimeContentPartDone": ("litellm.types.llms.openai", "OpenAIRealtimeContentPartDone"), - "OpenAIRealtimeConversationCreated": ("litellm.types.llms.openai", "OpenAIRealtimeConversationCreated"), - "OpenAIRealtimeConversationItemAdded": ("litellm.types.llms.openai", "OpenAIRealtimeConversationItemAdded"), - "OpenAIRealtimeConversationItemCreated": ("litellm.types.llms.openai", "OpenAIRealtimeConversationItemCreated"), - "OpenAIRealtimeConversationItemDone": ("litellm.types.llms.openai", "OpenAIRealtimeConversationItemDone"), - "OpenAIRealtimeConversationObject": ("litellm.types.llms.openai", "OpenAIRealtimeConversationObject"), - "OpenAIRealtimeDoneEvent": ("litellm.types.llms.openai", "OpenAIRealtimeDoneEvent"), - "OpenAIRealtimeEventTypes": ("litellm.types.llms.openai", "OpenAIRealtimeEventTypes"), - "OpenAIRealtimeEvents": ("litellm.assistants.main", "OpenAIRealtimeEvents"), - "OpenAIRealtimeFunctionCallArgumentsDone": ( - "litellm.types.llms.openai", - "OpenAIRealtimeFunctionCallArgumentsDone", - ), - "OpenAIRealtimeInputAudioBufferSpeechEvent": ( - "litellm.types.llms.openai", - "OpenAIRealtimeInputAudioBufferSpeechEvent", - ), - "OpenAIRealtimeInputAudioTranscriptionCompleted": ( - "litellm.types.llms.openai", - "OpenAIRealtimeInputAudioTranscriptionCompleted", - ), - "OpenAIRealtimeInputAudioTranscriptionDelta": ( - "litellm.types.llms.openai", - "OpenAIRealtimeInputAudioTranscriptionDelta", - ), - "OpenAIRealtimeOutputItemDone": ("litellm.types.llms.openai", "OpenAIRealtimeOutputItemDone"), - "OpenAIRealtimeResponseAudioDone": ("litellm.types.llms.openai", "OpenAIRealtimeResponseAudioDone"), - "OpenAIRealtimeResponseContentPart": ("litellm.types.llms.openai", "OpenAIRealtimeResponseContentPart"), - "OpenAIRealtimeResponseContentPartAdded": ( - "litellm.types.llms.openai", - "OpenAIRealtimeResponseContentPartAdded", - ), - "OpenAIRealtimeResponseDelta": ("litellm.types.llms.openai", "OpenAIRealtimeResponseDelta"), - "OpenAIRealtimeResponseDoneObject": ("litellm.types.llms.openai", "OpenAIRealtimeResponseDoneObject"), - "OpenAIRealtimeResponseTextDone": ("litellm.types.llms.openai", "OpenAIRealtimeResponseTextDone"), - "OpenAIRealtimeResponseUsage": ("litellm.types.llms.openai", "OpenAIRealtimeResponseUsage"), - "OpenAIRealtimeStreamList": ("litellm.assistants.main", "OpenAIRealtimeStreamList"), - "OpenAIRealtimeStreamResponseBaseObject": ( - "litellm.types.llms.openai", - "OpenAIRealtimeStreamResponseBaseObject", - ), - "OpenAIRealtimeStreamResponseOutputItem": ( - "litellm.types.llms.openai", - "OpenAIRealtimeStreamResponseOutputItem", - ), - "OpenAIRealtimeStreamResponseOutputItemAdded": ( - "litellm.types.llms.openai", - "OpenAIRealtimeStreamResponseOutputItemAdded", - ), - "OpenAIRealtimeStreamResponseOutputItemContent": ( - "litellm.types.llms.openai", - "OpenAIRealtimeStreamResponseOutputItemContent", - ), - "OpenAIRealtimeStreamSession": ("litellm.types.llms.openai", "OpenAIRealtimeStreamSession"), - "OpenAIRealtimeStreamSessionEvents": ("litellm.types.llms.openai", "OpenAIRealtimeStreamSessionEvents"), - "OpenAIRealtimeTurnDetection": ("litellm.types.llms.openai", "OpenAIRealtimeTurnDetection"), - "OpenAIRealtimeUsageTokenDetails": ("litellm.types.llms.openai", "OpenAIRealtimeUsageTokenDetails"), - "OpenAITextCompletion": ("litellm.llms.openai.completion.handler", "OpenAITextCompletion"), - "OpenAITextCompletionUserMessage": ("litellm.types.llms.openai", "OpenAITextCompletionUserMessage"), - "OpenAIVideoObject": ("litellm.types.llms.openai", "OpenAIVideoObject"), - "OpenAIWebSearchOptions": ("litellm.types.llms.openai", "OpenAIWebSearchOptions"), - "OpenAIWebSearchUserLocation": ("litellm.types.llms.openai", "OpenAIWebSearchUserLocation"), - "OpenAIWebSearchUserLocationApproximate": ( - "litellm.types.llms.openai", - "OpenAIWebSearchUserLocationApproximate", - ), - "OptionalPreCallChecks": ("litellm.files.main", "OptionalPreCallChecks"), - "OutputCodeInterpreterCall": ("litellm.types.responses.main", "OutputCodeInterpreterCall"), - "OutputCodeInterpreterCallLog": ("litellm.types.responses.main", "OutputCodeInterpreterCallLog"), - "OutputFunctionToolCall": ("litellm.types.responses.main", "OutputFunctionToolCall"), - "OutputImageGenerationCall": ("litellm.types.responses.main", "OutputImageGenerationCall"), - "OutputItemAddedEvent": ("litellm.types.llms.openai", "OutputItemAddedEvent"), - "OutputItemDoneEvent": ("litellm.types.llms.openai", "OutputItemDoneEvent"), - "OutputText": ("litellm.types.responses.main", "OutputText"), - "OutputTextAnnotationAddedEvent": ("litellm.types.llms.openai", "OutputTextAnnotationAddedEvent"), - "OutputTextDeltaEvent": ("litellm.types.llms.openai", "OutputTextDeltaEvent"), - "OutputTextDoneEvent": ("litellm.types.llms.openai", "OutputTextDoneEvent"), - "OutputTokensDetails": ("litellm.types.llms.openai", "OutputTokensDetails"), - "PART_UNION_TYPES": ("litellm.assistants.main", "PART_UNION_TYPES"), - "PalmConfig": ("litellm.llms.deprecated_providers.palm", "PalmConfig"), - "PathLike": ("litellm.assistants.main", "PathLike"), - "PermissionDeniedError": ("litellm.exceptions", "PermissionDeniedError"), - "Phase": ("litellm.responses.main", "Phase"), - "PreRoutingHookResponse": ("litellm.types.router", "PreRoutingHookResponse"), - "PreRoutingStrategy": ("litellm.types.router", "PreRoutingStrategy"), - "PredibaseChatCompletion": ("litellm.llms.predibase.chat.handler", "PredibaseChatCompletion"), - "PrivateAttr": ("litellm.responses.main", "PrivateAttr"), - "PromptCacheBreakpoint": ("litellm.types.llms.openai", "PromptCacheBreakpoint"), - "PromptCacheOptions": ("litellm.types.llms.openai", "PromptCacheOptions"), - "PromptObject": ("litellm.types.llms.openai", "PromptObject"), - "PromptSpec": ("litellm.types.prompts.init_prompts", "PromptSpec"), - "PromptTokensDetails": ("litellm.main", "PromptTokensDetails"), - "Protocol": ("litellm.files.main", "Protocol"), - "ProviderConfigManager": ("litellm.utils", "ProviderConfigManager"), - "ProviderSpecificHeader": ("litellm.types.utils", "ProviderSpecificHeader"), - "ProviderSpecificHeaderUtils": ( - "litellm.litellm_core_utils.get_provider_specific_headers", - "ProviderSpecificHeaderUtils", - ), - "REASONING_EFFORT": ("litellm.assistants.main", "REASONING_EFFORT"), - "RateLimitError": ("litellm.exceptions", "RateLimitError"), - "RateLimitErrorCategory": ("litellm.exceptions", "RateLimitErrorCategory"), - "RateLimitType": ("litellm.exceptions", "RateLimitType"), - "RawRequestTypedDict": ("litellm.types.utils", "RawRequestTypedDict"), - "ReadOnly": ("litellm.files.main", "ReadOnly"), - "Reasoning": ("litellm.responses.main", "Reasoning"), - "ReasoningSummaryPartDoneEvent": ("litellm.types.llms.openai", "ReasoningSummaryPartDoneEvent"), - "ReasoningSummaryTextDeltaEvent": ("litellm.types.llms.openai", "ReasoningSummaryTextDeltaEvent"), - "ReasoningSummaryTextDoneEvent": ("litellm.types.llms.openai", "ReasoningSummaryTextDoneEvent"), - "RefusalDeltaEvent": ("litellm.types.llms.openai", "RefusalDeltaEvent"), - "RefusalDoneEvent": ("litellm.types.llms.openai", "RefusalDoneEvent"), - "RequestType": ("litellm.types.router", "RequestType"), - "Required": ("litellm.files.main", "Required"), - "Response": ("litellm.assistants.main", "Response"), - "ResponseAPIUsage": ("litellm.types.llms.openai", "ResponseAPIUsage"), - "ResponseCompletedEvent": ("litellm.types.llms.openai", "ResponseCompletedEvent"), - "ResponseCreatedEvent": ("litellm.types.llms.openai", "ResponseCreatedEvent"), - "ResponseFailedEvent": ("litellm.types.llms.openai", "ResponseFailedEvent"), - "ResponseFunctionToolCall": ("litellm.responses.main", "ResponseFunctionToolCall"), - "ResponseInProgressEvent": ("litellm.types.llms.openai", "ResponseInProgressEvent"), - "ResponseIncludable": ("litellm.responses.main", "ResponseIncludable"), - "ResponseIncompleteEvent": ("litellm.types.llms.openai", "ResponseIncompleteEvent"), - "ResponseInputParam": ("litellm.responses.main", "ResponseInputParam"), - "ResponseOutputItem": ("litellm.assistants.main", "ResponseOutputItem"), - "ResponsePartAddedEvent": ("litellm.types.llms.openai", "ResponsePartAddedEvent"), - "ResponseText": ("litellm.responses.main", "ResponseText"), - "ResponsesAPIOptionalRequestParams": ("litellm.types.llms.openai", "ResponsesAPIOptionalRequestParams"), - "ResponsesAPIRequestParams": ("litellm.types.llms.openai", "ResponsesAPIRequestParams"), - "ResponsesAPIRequestUtils": ("litellm.responses.utils", "ResponsesAPIRequestUtils"), - "ResponsesAPIResponse": ("litellm.types.llms.openai", "ResponsesAPIResponse"), - "ResponsesAPIStatus": ("litellm.assistants.main", "ResponsesAPIStatus"), - "ResponsesAPIStreamEvents": ("litellm.types.llms.openai", "ResponsesAPIStreamEvents"), - "ResponsesAPIStreamOptions": ("litellm.types.llms.openai", "ResponsesAPIStreamOptions"), - "ResponsesAPIStreamingResponse": ("litellm.assistants.main", "ResponsesAPIStreamingResponse"), - "ResponsesToolUsage": ("litellm.types.llms.openai", "ResponsesToolUsage"), - "RetrieveBatchRequest": ("litellm.types.llms.openai", "RetrieveBatchRequest"), - "RetryPolicy": ("litellm.types.router", "RetryPolicy"), - "Router": ("litellm.router", "Router"), - "RouterCacheEnum": ("litellm.types.router", "RouterCacheEnum"), - "RouterConfig": ("litellm.types.router", "RouterConfig"), - "RouterErrors": ("litellm.types.router", "RouterErrors"), - "RouterGeneralSettings": ("litellm.types.router", "RouterGeneralSettings"), - "RouterModelGroupAliasItem": ("litellm.types.router", "RouterModelGroupAliasItem"), - "RouterRateLimitError": ("litellm.types.router", "RouterRateLimitError"), - "RouterRateLimitErrorBasic": ("litellm.types.router", "RouterRateLimitErrorBasic"), - "RoutingContext": ("litellm.types.router", "RoutingContext"), - "RoutingGroup": ("litellm.types.router", "RoutingGroup"), - "RoutingPlugin": ("litellm.types.router", "RoutingPlugin"), - "RoutingStrategy": ("litellm.types.router", "RoutingStrategy"), - "Run": ("litellm.assistants.main", "Run"), - "SPECIAL_MODEL_INFO_PARAMS": ("litellm.files.main", "SPECIAL_MODEL_INFO_PARAMS"), - "SagemakerChatHandler": ("litellm.llms.sagemaker.chat.handler", "SagemakerChatHandler"), - "SagemakerLLM": ("litellm.llms.sagemaker.completion.handler", "SagemakerLLM"), - "Scheduler": ("litellm.scheduler", "Scheduler"), - "SchedulerCacheKeys": ("litellm.scheduler", "SchedulerCacheKeys"), - "SearchProvider": ("litellm.files.main", "SearchProvider"), - "SearchResponse": ("litellm.llms.base_llm.search.transformation", "SearchResponse"), - "SearchToolInfoTypedDict": ("litellm.types.router", "SearchToolInfoTypedDict"), - "SearchToolLiteLLMParams": ("litellm.types.router", "SearchToolLiteLLMParams"), - "SearchToolTypedDict": ("litellm.types.router", "SearchToolTypedDict"), - "SerializerFunctionWrapHandler": ("litellm.assistants.main", "SerializerFunctionWrapHandler"), - "ServiceUnavailableError": ("litellm.exceptions", "ServiceUnavailableError"), - "ShellToolParam": ("litellm.types.llms.openai", "ShellToolParam"), - "SlackAlerting": ("litellm.integrations.SlackAlerting.slack_alerting", "SlackAlerting"), - "StandardLoggingRoutingDecision": ("litellm.types.utils", "StandardLoggingRoutingDecision"), - "StreamingChoices": ("litellm.types.utils", "StreamingChoices"), - "SyncCursorPage": ("litellm.assistants.main", "SyncCursorPage"), - "TaggedPreRoutingStrategy": ("litellm.types.router", "TaggedPreRoutingStrategy"), - "TextChoices": ("litellm.types.utils", "TextChoices"), - "TextCompletionStreamWrapper": ("litellm.utils", "TextCompletionStreamWrapper"), - "Thread": ("litellm.types.llms.openai", "Thread"), - "ThreadPoolExecutor": ("litellm.batch_completion.main", "ThreadPoolExecutor"), - "Timeout": ("litellm.exceptions", "Timeout"), - "TogetherAIRerank": ("litellm.llms.together_ai.rerank.handler", "TogetherAIRerank"), - "Tool": ("litellm.assistants.main", "Tool"), - "ToolChoice": ("litellm.responses.main", "ToolChoice"), - "ToolMessageContentPart": ("litellm.assistants.main", "ToolMessageContentPart"), - "ToolParam": ("litellm.responses.main", "ToolParam"), - "ToolResourcesCodeInterpreter": ("litellm.types.llms.openai", "ToolResourcesCodeInterpreter"), - "ToolResourcesFileSearch": ("litellm.types.llms.openai", "ToolResourcesFileSearch"), - "ToolResourcesFileSearchVectorStore": ("litellm.types.llms.openai", "ToolResourcesFileSearchVectorStore"), - "TopazModelInfo": ("litellm.llms.topaz.common_utils", "TopazModelInfo"), - "TypeAlias": ("litellm.assistants.main", "TypeAlias"), - "TypeVar": ("litellm.files.main", "TypeVar"), - "TypedDict": ("litellm.files.main", "TypedDict"), - "UnprocessableEntityError": ("litellm.exceptions", "UnprocessableEntityError"), - "UnsupportedParamsError": ("litellm.exceptions", "UnsupportedParamsError"), - "UpdateRouterConfig": ("litellm.types.router", "UpdateRouterConfig"), - "Usage": ("litellm.types.utils", "Usage"), - "VALID_LITELLM_ENVIRONMENTS": ("litellm.files.main", "VALID_LITELLM_ENVIRONMENTS"), - "ValidAssistantMessageContentTypes": ("litellm.assistants.main", "ValidAssistantMessageContentTypes"), - "ValidAssistantMessageContentTypesLiteral": ( - "litellm.assistants.main", - "ValidAssistantMessageContentTypesLiteral", - ), - "ValidChatCompletionMessageContentTypes": ("litellm.assistants.main", "ValidChatCompletionMessageContentTypes"), - "ValidChatCompletionMessageContentTypesLiteral": ( - "litellm.assistants.main", - "ValidChatCompletionMessageContentTypesLiteral", - ), - "ValidUserMessageContentTypes": ("litellm.assistants.main", "ValidUserMessageContentTypes"), - "ValidUserMessageContentTypesLiteral": ("litellm.assistants.main", "ValidUserMessageContentTypesLiteral"), - "VectorStoreIndexRegistry": ("litellm.vector_stores.vector_store_registry", "VectorStoreIndexRegistry"), - "VectorStoreRegistry": ("litellm.vector_stores.vector_store_registry", "VectorStoreRegistry"), - "VertexAIBatchPrediction": ("litellm.llms.vertex_ai.batches.handler", "VertexAIBatchPrediction"), - "VertexAIFilesHandler": ("litellm.llms.vertex_ai.files.handler", "VertexAIFilesHandler"), - "VertexAIGemmaModels": ("litellm.llms.vertex_ai.vertex_gemma_models.main", "VertexAIGemmaModels"), - "VertexAIModelGardenModels": ("litellm.llms.vertex_ai.vertex_model_garden.main", "VertexAIModelGardenModels"), - "VertexAIModelRoute": ("litellm.llms.vertex_ai.common_utils", "VertexAIModelRoute"), - "VertexAIPartnerModels": ("litellm.llms.vertex_ai.vertex_ai_partner_models.main", "VertexAIPartnerModels"), - "VertexAITextEmbeddingConfig": ( - "litellm.llms.vertex_ai.vertex_embeddings.transformation", - "VertexAITextEmbeddingConfig", - ), - "VertexEmbedding": ("litellm.llms.vertex_ai.vertex_embeddings.embedding_handler", "VertexEmbedding"), - "VertexFineTuningAPI": ("litellm.llms.vertex_ai.fine_tuning.handler", "VertexFineTuningAPI"), - "VertexImageGeneration": ( - "litellm.llms.vertex_ai.image_generation.image_generation_handler", - "VertexImageGeneration", - ), - "VertexLLM": ("litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", "VertexLLM"), - "VertexMultimodalEmbedding": ( - "litellm.llms.vertex_ai.multimodal_embeddings.embedding_handler", - "VertexMultimodalEmbedding", - ), - "VideoCreateOptionalRequestParams": ("litellm.types.videos.main", "VideoCreateOptionalRequestParams"), - "VideoGenerationRequestUtils": ("litellm.videos.utils", "VideoGenerationRequestUtils"), - "VideoObject": ("litellm.types.videos.main", "VideoObject"), - "WatsonXChatHandler": ("litellm.llms.watsonx.chat.handler", "WatsonXChatHandler"), - "WebSearchCallCompletedEvent": ("litellm.types.llms.openai", "WebSearchCallCompletedEvent"), - "WebSearchCallInProgressEvent": ("litellm.types.llms.openai", "WebSearchCallInProgressEvent"), - "WebSearchCallSearchingEvent": ("litellm.types.llms.openai", "WebSearchCallSearchingEvent"), - "WebSearchOptions": ("litellm.types.llms.openai", "WebSearchOptions"), - "WebSearchOptionsUserLocation": ("litellm.types.llms.openai", "WebSearchOptionsUserLocation"), - "WebSearchOptionsUserLocationApproximate": ( - "litellm.types.llms.openai", - "WebSearchOptionsUserLocationApproximate", - ), - "WebSearchToolUsage": ("litellm.types.llms.openai", "WebSearchToolUsage"), - "XAIModelInfo": ("litellm.llms.xai.common_utils", "XAIModelInfo"), - "_arealtime": ("litellm.realtime_api.main", "_arealtime"), - "_aresponses_websocket": ("litellm.responses.main", "_aresponses_websocket"), - "a_add_message": ("litellm.assistants.main", "a_add_message"), - "aadapter_completion": ("litellm.main", "aadapter_completion"), - "aadapter_generate_content": ("litellm.main", "aadapter_generate_content"), - "acancel_batch": ("litellm.batches.main", "acancel_batch"), - "acancel_fine_tuning_job": ("litellm.fine_tuning.main", "acancel_fine_tuning_job"), - "acancel_responses": ("litellm.responses.main", "acancel_responses"), - "acode_interpreter_tool": ("litellm.sandbox.main", "acode_interpreter_tool"), - "acompact_responses": ("litellm.responses.main", "acompact_responses"), - "acompletion": ("litellm.main", "acompletion"), - "acompletion_with_retries": ("litellm.main", "acompletion_with_retries"), - "acount_tokens": ("litellm.main", "acount_tokens"), - "acreate_agent": ("litellm.interactions.agents.main", "acreate"), - "acreate_assistants": ("litellm.assistants.main", "acreate_assistants"), - "acreate_batch": ("litellm.batches.main", "acreate_batch"), - "acreate_container": ("litellm.containers.main", "acreate_container"), - "acreate_file": ("litellm.files.main", "acreate_file"), - "acreate_fine_tuning_job": ("litellm.fine_tuning.main", "acreate_fine_tuning_job"), - "acreate_realtime_client_secret": ("litellm.realtime_api.main", "acreate_realtime_client_secret"), - "acreate_realtime_transcription_session": ( - "litellm.realtime_api.main", - "acreate_realtime_transcription_session", - ), - "acreate_sandbox": ("litellm.sandbox.main", "acreate_sandbox"), - "acreate_skill": ("litellm.skills.main", "acreate_skill"), - "acreate_thread": ("litellm.assistants.main", "acreate_thread"), - "adapter_completion": ("litellm.main", "adapter_completion"), - "add_message": ("litellm.assistants.main", "add_message"), - "add_provider_specific_params_to_optional_params": ( - "litellm.utils", - "add_provider_specific_params_to_optional_params", - ), - "add_system_prompt_to_messages": ( - "litellm.litellm_core_utils.prompt_templates.common_utils", - "add_system_prompt_to_messages", - ), - "add_trusted_model_credentials_to_litellm_params": ( - "litellm.litellm_core_utils.get_litellm_params", - "add_trusted_model_credentials_to_litellm_params", - ), - "adelete_agent": ("litellm.interactions.agents.main", "adelete"), - "adelete_assistant": ("litellm.assistants.main", "adelete_assistant"), - "adelete_container": ("litellm.containers.main", "adelete_container"), - "adelete_responses": ("litellm.responses.main", "adelete_responses"), - "adelete_sandbox": ("litellm.sandbox.main", "adelete_sandbox"), - "adelete_skill": ("litellm.skills.main", "adelete_skill"), - "aembedding": ("litellm.main", "aembedding"), - "afile_content": ("litellm.files.main", "afile_content"), - "afile_delete": ("litellm.files.main", "afile_delete"), - "afile_list": ("litellm.files.main", "afile_list"), - "afile_retrieve": ("litellm.files.main", "afile_retrieve"), - "agenerate_content": ("litellm.google_genai.main", "agenerate_content"), - "aget_agent": ("litellm.interactions.agents.main", "aget"), - "aget_assistants": ("litellm.assistants.main", "aget_assistants"), - "aget_messages": ("litellm.assistants.main", "aget_messages"), - "aget_responses": ("litellm.responses.main", "aget_responses"), - "aget_skill": ("litellm.skills.main", "aget_skill"), - "aget_thread": ("litellm.assistants.main", "aget_thread"), - "ahealth_check": ("litellm.main", "ahealth_check"), - "aimage_edit": ("litellm.images.main", "aimage_edit"), - "aimage_generation": ("litellm.images.main", "aimage_generation"), - "aimage_variation": ("litellm.images.main", "aimage_variation"), - "aingest": ("litellm.rag.main", "aingest"), - "alist_agent_versions": ("litellm.interactions.agents.main", "alist_versions"), - "alist_agents": ("litellm.interactions.agents.main", "alist"), - "alist_batches": ("litellm.batches.main", "alist_batches"), - "alist_container_files": ("litellm.containers.main", "alist_container_files"), - "alist_containers": ("litellm.containers.main", "alist_containers"), - "alist_fine_tuning_jobs": ("litellm.fine_tuning.main", "alist_fine_tuning_jobs"), - "alist_input_items": ("litellm.responses.main", "alist_input_items"), - "alist_skills": ("litellm.skills.main", "alist_skills"), - "allm_passthrough_route": ("litellm.passthrough.main", "allm_passthrough_route"), - "amoderation": ("litellm.main", "amoderation"), - "anthropic_batches_instance": ("litellm.batches.main", "anthropic_batches_instance"), - "anthropic_chat_completions": ("litellm.main", "anthropic_chat_completions"), - "anthropic_messages": ( - "litellm.llms.anthropic.experimental_pass_through.messages.handler", - "anthropic_messages", - ), - "anthropic_messages_handler": ( - "litellm.llms.anthropic.experimental_pass_through.messages.handler", - "anthropic_messages_handler", - ), - "aocr": ("litellm.ocr.main", "aocr"), - "aquery": ("litellm.rag.main", "aquery"), - "arealtime_calls": ("litellm.realtime_api.main", "arealtime_calls"), - "arerank": ("litellm.rerank_api.main", "arerank"), - "aresponses": ("litellm.responses.main", "aresponses"), - "aresponses_api_with_mcp": ("litellm.responses.main", "aresponses_api_with_mcp"), - "aresponses_with_retries": ("litellm.main", "aresponses_with_retries"), - "aretrieve_batch": ("litellm.batches.main", "aretrieve_batch"), - "aretrieve_container": ("litellm.containers.main", "aretrieve_container"), - "aretrieve_fine_tuning_job": ("litellm.fine_tuning.main", "aretrieve_fine_tuning_job"), - "arun_code": ("litellm.sandbox.main", "arun_code"), - "arun_thread": ("litellm.assistants.main", "arun_thread"), - "arun_thread_stream": ("litellm.assistants.main", "arun_thread_stream"), - "asearch": ("litellm.search.main", "asearch"), - "aspeech": ("litellm.main", "aspeech"), - "async_completion_with_fallbacks": ( - "litellm.litellm_core_utils.fallback_utils", - "async_completion_with_fallbacks", - ), - "async_mock_completion_streaming_obj": ("litellm.utils", "async_mock_completion_streaming_obj"), - "atext_completion": ("litellm.main", "atext_completion"), - "atranscription": ("litellm.main", "atranscription"), - "aupload_container_file": ("litellm.containers.main", "aupload_container_file"), - "avector_store_file_content": ("litellm.vector_store_files.main", "aretrieve_content"), - "avector_store_file_create": ("litellm.vector_store_files.main", "acreate"), - "avector_store_file_delete": ("litellm.vector_store_files.main", "adelete"), - "avector_store_file_list": ("litellm.vector_store_files.main", "alist"), - "avector_store_file_retrieve": ("litellm.vector_store_files.main", "aretrieve"), - "avector_store_file_update": ("litellm.vector_store_files.main", "aupdate"), - "avideo_content": ("litellm.videos.main", "avideo_content"), - "avideo_create_character": ("litellm.videos.main", "avideo_create_character"), - "avideo_edit": ("litellm.videos.main", "avideo_edit"), - "avideo_extension": ("litellm.videos.main", "avideo_extension"), - "avideo_generation": ("litellm.videos.main", "avideo_generation"), - "avideo_get_character": ("litellm.videos.main", "avideo_get_character"), - "avideo_list": ("litellm.videos.main", "avideo_list"), - "avideo_remix": ("litellm.videos.main", "avideo_remix"), - "avideo_status": ("litellm.videos.main", "avideo_status"), - "azure_ai_embedding": ("litellm.main", "azure_ai_embedding"), - "azure_anthropic_chat_completions": ("litellm.main", "azure_anthropic_chat_completions"), - "azure_assistants_api": ("litellm.assistants.main", "azure_assistants_api"), - "azure_audio_transcriptions": ("litellm.main", "azure_audio_transcriptions"), - "azure_batches_instance": ("litellm.batches.main", "azure_batches_instance"), - "azure_chat_completions": ("litellm.images.main", "azure_chat_completions"), - "azure_files_instance": ("litellm.files.main", "azure_files_instance"), - "azure_fine_tuning_apis_instance": ("litellm.fine_tuning.main", "azure_fine_tuning_apis_instance"), - "azure_o1_chat_completions": ("litellm.main", "azure_o1_chat_completions"), - "azure_text_completions": ("litellm.main", "azure_text_completions"), - "base_llm_aiohttp_handler": ("litellm.images.main", "base_llm_aiohttp_handler"), - "base_llm_http_handler": ("litellm.files.main", "base_llm_http_handler"), - "batch_completion": ("litellm.batch_completion.main", "batch_completion"), - "batch_completion_models": ("litellm.batch_completion.main", "batch_completion_models"), - "batch_completion_models_all_responses": ( - "litellm.batch_completion.main", - "batch_completion_models_all_responses", - ), - "bedrock_converse_chat_completion": ("litellm.main", "bedrock_converse_chat_completion"), - "bedrock_embedding": ("litellm.main", "bedrock_embedding"), - "bedrock_files_instance": ("litellm.files.main", "bedrock_files_instance"), - "bedrock_image_edit": ("litellm.images.main", "bedrock_image_edit"), - "bedrock_image_generation": ("litellm.images.main", "bedrock_image_generation"), - "bedrock_rerank": ("litellm.rerank_api.main", "bedrock_rerank"), - "bfl_image_edit": ("litellm.llms.black_forest_labs.image_edit.handler", "bfl_image_edit"), - "bfl_image_generation": ("litellm.llms.black_forest_labs.image_generation.handler", "bfl_image_generation"), - "build_code_interpreter_log_outputs": ("litellm.types.responses.main", "build_code_interpreter_log_outputs"), - "bytez_transformation": ("litellm.main", "bytez_transformation"), - "calculate_request_duration": ("litellm.litellm_core_utils.audio_utils.utils", "calculate_request_duration"), - "cancel_batch": ("litellm.batches.main", "cancel_batch"), - "cancel_fine_tuning_job": ("litellm.fine_tuning.main", "cancel_fine_tuning_job"), - "cancel_responses": ("litellm.responses.main", "cancel_responses"), - "cast": ("litellm.files.main", "cast"), - "client": ("litellm.utils", "client"), - "close_litellm_async_clients": ( - "litellm.llms.custom_httpx.async_client_cleanup", - "close_litellm_async_clients", - ), - "codestral_text_completions": ("litellm.main", "codestral_text_completions"), - "compact_responses": ("litellm.responses.main", "compact_responses"), - "completion": ("litellm.main", "completion"), - "completion_with_fallbacks": ("litellm.litellm_core_utils.fallback_utils", "completion_with_fallbacks"), - "completion_with_retries": ("litellm.main", "completion_with_retries"), - "compress": ("litellm.compression.compress", "compress"), - "config_completion": ("litellm.main", "config_completion"), - "contextmanager": ("litellm.responses.main", "contextmanager"), - "convert_file_document_to_url_document": ("litellm.ocr.main", "convert_file_document_to_url_document"), - "convert_model_response_to_streaming": ( - "litellm.llms.base_llm.base_model_iterator", - "convert_model_response_to_streaming", - ), - "create_agent": ("litellm.interactions.agents.main", "create"), - "create_assistants": ("litellm.assistants.main", "create_assistants"), - "create_batch": ("litellm.batches.main", "create_batch"), - "create_container": ("litellm.containers.main", "create_container"), - "create_file": ("litellm.files.main", "create_file"), - "create_fine_tuning_job": ("litellm.fine_tuning.main", "create_fine_tuning_job"), - "create_skill": ("litellm.skills.main", "create_skill"), - "create_thread": ("litellm.assistants.main", "create_thread"), - "custom_chat_llm_router": ("litellm.llms.custom_llm", "custom_chat_llm_router"), - "custom_prompt": ("litellm.litellm_core_utils.prompt_templates.factory", "custom_prompt"), - "databricks_embedding": ("litellm.main", "databricks_embedding"), - "dataclass": ("litellm.files.main", "dataclass"), - "decode_video_id_with_provider": ("litellm.types.videos.utils", "decode_video_id_with_provider"), - "declared_authenticating_provider": ( - "litellm.litellm_core_utils.get_llm_provider_logic", - "declared_authenticating_provider", - ), - "deepcopy": ("litellm.main", "deepcopy"), - "delete_agent": ("litellm.interactions.agents.main", "delete"), - "delete_assistant": ("litellm.assistants.main", "delete_assistant"), - "delete_container": ("litellm.containers.main", "delete_container"), - "delete_responses": ("litellm.responses.main", "delete_responses"), - "delete_skill": ("litellm.skills.main", "delete_skill"), - "disable_cache": ("litellm.caching.caching", "disable_cache"), - "embedding": ("litellm.main", "embedding"), - "enable_cache": ("litellm.caching.caching", "enable_cache"), - "field_serializer": ("litellm.assistants.main", "field_serializer"), - "field_validator": ("litellm.files.main", "field_validator"), - "file_content": ("litellm.files.main", "file_content"), - "file_content_streaming": ("litellm.files.main", "file_content_streaming"), - "file_delete": ("litellm.files.main", "file_delete"), - "file_list": ("litellm.files.main", "file_list"), - "file_retrieve": ("litellm.files.main", "file_retrieve"), - "filter_out_litellm_params": ("litellm.utils", "filter_out_litellm_params"), - "flatten_form_field_values": ("litellm.litellm_core_utils.llm_request_utils", "flatten_form_field_values"), - "flatten_unencrypted_web_search_results_in_anthropic_messages": ( - "litellm.llms.anthropic.common_utils", - "flatten_unencrypted_web_search_results_in_anthropic_messages", - ), - "function_call_prompt": ("litellm.litellm_core_utils.prompt_templates.factory", "function_call_prompt"), - "gdc_transformation": ("litellm.main", "gdc_transformation"), - "get_agent": ("litellm.interactions.agents.main", "get"), - "get_api_key_from_env": ("litellm.llms.gemini.common_utils", "get_api_key_from_env"), - "get_assistants": ("litellm.assistants.main", "get_assistants"), - "get_audio_file_for_health_check": ( - "litellm.litellm_core_utils.audio_utils.utils", - "get_audio_file_for_health_check", - ), - "get_azure_credentials": ("litellm.llms.azure.common_utils", "get_azure_credentials"), - "get_completion_messages": ( - "litellm.litellm_core_utils.prompt_templates.common_utils", - "get_completion_messages", - ), - "get_configured_request_timeout": ( - "litellm.litellm_core_utils.request_timeout_resolver", - "get_configured_request_timeout", - ), - "get_content_from_model_response": ( - "litellm.litellm_core_utils.prompt_templates.common_utils", - "get_content_from_model_response", - ), - "get_litellm_gateway_api_key": ("litellm.litellm_core_utils.cli_token_utils", "get_litellm_gateway_api_key"), - "get_messages": ("litellm.assistants.main", "get_messages"), - "get_messages_interceptors": ( - "litellm.llms.anthropic.experimental_pass_through.messages.interceptors", - "get_messages_interceptors", - ), - "get_mime_type": ("litellm.ocr.main", "get_mime_type"), - "get_non_default_completion_params": ("litellm.utils", "get_non_default_completion_params"), - "get_non_default_transcription_params": ("litellm.utils", "get_non_default_transcription_params"), - "get_openai_credentials": ("litellm.llms.openai.common_utils", "get_openai_credentials"), - "get_optional_params_add_message": ("litellm.assistants.utils", "get_optional_params_add_message"), - "get_optional_params_embeddings": ("litellm.utils", "get_optional_params_embeddings"), - "get_optional_params_image_gen": ("litellm.utils", "get_optional_params_image_gen"), - "get_optional_params_transcription": ("litellm.utils", "get_optional_params_transcription"), - "get_optional_rerank_params": ("litellm.rerank_api.rerank_utils", "get_optional_rerank_params"), - "get_requester_metadata": ("litellm.utils", "get_requester_metadata"), - "get_responses": ("litellm.responses.main", "get_responses"), - "get_secret": ("litellm.secret_managers.main", "get_secret"), - "get_secret_bool": ("litellm.secret_managers.main", "get_secret_bool"), - "get_secret_str": ("litellm.secret_managers.main", "get_secret_str"), - "get_skill": ("litellm.skills.main", "get_skill"), - "get_standard_openai_params": ("litellm.utils", "get_standard_openai_params"), - "get_thread": ("litellm.assistants.main", "get_thread"), - "get_type_hints": ("litellm.files.main", "get_type_hints"), - "get_vertex_ai_model_route": ("litellm.llms.vertex_ai.common_utils", "get_vertex_ai_model_route"), - "google_batch_embeddings": ("litellm.main", "google_batch_embeddings"), - "groq_chat_completions": ("litellm.main", "groq_chat_completions"), - "heroku_transformation": ("litellm.main", "heroku_transformation"), - "huggingface_embed": ("litellm.main", "huggingface_embed"), - "image_edit": ("litellm.images.main", "image_edit"), - "image_generation": ("litellm.images.main", "image_generation"), - "image_variation": ("litellm.images.main", "image_variation"), - "infer_openai_data_residency": ("litellm.llms.openai.data_residency", "infer_openai_data_residency"), - "ingest": ("litellm.rag.main", "ingest"), - "is_azure_document_intelligence_model": ( - "litellm.llms.azure_ai.ocr.common_utils", - "is_azure_document_intelligence_model", - ), - "is_reasoning_auto_summary_enabled": ( - "litellm.llms.anthropic.experimental_pass_through.utils", - "is_reasoning_auto_summary_enabled", - ), - "lemonade_transformation": ("litellm.main", "lemonade_transformation"), - "list_agent_versions": ("litellm.interactions.agents.main", "list_versions"), - "list_agents": ("litellm.interactions.agents.main", "list"), - "list_batches": ("litellm.batches.main", "list_batches"), - "list_container_files": ("litellm.containers.main", "list_container_files"), - "list_containers": ("litellm.containers.main", "list_containers"), - "list_fine_tuning_jobs": ("litellm.fine_tuning.main", "list_fine_tuning_jobs"), - "list_input_items": ("litellm.responses.main", "list_input_items"), - "list_skills": ("litellm.skills.main", "list_skills"), - "litellm_completion_transformation_handler": ( - "litellm.responses.main", - "litellm_completion_transformation_handler", - ), - "llm_http_handler": ("litellm.videos.main", "llm_http_handler"), - "llm_passthrough_route": ("litellm.passthrough.main", "llm_passthrough_route"), - "map_system_message_pt": ("litellm.litellm_core_utils.prompt_templates.factory", "map_system_message_pt"), - "maybe_run_chat_completion_agentic_loop": ( - "litellm.litellm_core_utils.chat_completion_agentic_loop", - "maybe_run_chat_completion_agentic_loop", - ), - "mock_completion": ("litellm.main", "mock_completion"), - "mock_completion_streaming_obj": ("litellm.utils", "mock_completion_streaming_obj"), - "mock_embedding": ("litellm.litellm_core_utils.mock_functions", "mock_embedding"), - "mock_image_generation": ("litellm.litellm_core_utils.mock_functions", "mock_image_generation"), - "mock_response": ("litellm.llms.anthropic.experimental_pass_through.messages.utils", "mock_response"), - "mock_responses_api_response": ("litellm.responses.main", "mock_responses_api_response"), - "model_serializer": ("litellm.assistants.main", "model_serializer"), - "model_validator": ("litellm.files.main", "model_validator"), - "moderation": ("litellm.main", "moderation"), - "nlp_cloud_chat_completion": ("litellm.main", "nlp_cloud_chat_completion"), - "nvidia_riva_audio_transcriptions": ("litellm.main", "nvidia_riva_audio_transcriptions"), - "oci_transformation": ("litellm.main", "oci_transformation"), - "ocr": ("litellm.ocr.main", "ocr"), - "ollama_pt": ("litellm.litellm_core_utils.prompt_templates.factory", "ollama_pt"), - "openai_assistants_api": ("litellm.assistants.main", "openai_assistants_api"), - "openai_audio_transcriptions": ("litellm.main", "openai_audio_transcriptions"), - "openai_batches_instance": ("litellm.batches.main", "openai_batches_instance"), - "openai_chat_completions": ("litellm.images.main", "openai_chat_completions"), - "openai_files_instance": ("litellm.files.main", "openai_files_instance"), - "openai_fine_tuning_apis_instance": ("litellm.fine_tuning.main", "openai_fine_tuning_apis_instance"), - "openai_image_variations": ("litellm.images.main", "openai_image_variations"), - "openai_like_chat_completion": ("litellm.main", "openai_like_chat_completion"), - "openai_like_embedding": ("litellm.main", "openai_like_embedding"), - "openai_text_completions": ("litellm.main", "openai_text_completions"), - "override": ("litellm.assistants.main", "override"), - "ovhcloud_transformation": ("litellm.main", "ovhcloud_transformation"), - "parse_ocr_request_format": ("litellm.llms.base_llm.ocr.transformation", "parse_ocr_request_format"), - "partial": ("litellm.files.main", "partial"), - "peek_reasoning_summary_aliases": ("litellm.utils", "peek_reasoning_summary_aliases"), - "pre_process_non_default_params": ("litellm.utils", "pre_process_non_default_params"), - "predibase_chat_completions": ("litellm.main", "predibase_chat_completions"), - "print_verbose": ("litellm.main", "print_verbose"), - "prompt_factory": ("litellm.litellm_core_utils.prompt_templates.factory", "prompt_factory"), - "query": ("litellm.rag.main", "query"), - "read_config_args": ("litellm.utils", "read_config_args"), - "replicate_chat_completion": ("litellm.main", "replicate_chat_completion"), - "rerank": ("litellm.rerank_api.main", "rerank"), - "responses": ("litellm.responses.main", "responses"), - "responses_api_bridge_check": ("litellm.main", "responses_api_bridge_check"), - "responses_with_retries": ("litellm.main", "responses_with_retries"), - "retrieve_batch": ("litellm.batches.main", "retrieve_batch"), - "retrieve_container": ("litellm.containers.main", "retrieve_container"), - "retrieve_fine_tuning_job": ("litellm.fine_tuning.main", "retrieve_fine_tuning_job"), - "run_async_function": ("litellm.litellm_core_utils.asyncify", "run_async_function"), - "run_server": ("litellm.proxy.proxy_cli", "run_server"), - "run_thread": ("litellm.assistants.main", "run_thread"), - "run_thread_stream": ("litellm.assistants.main", "run_thread_stream"), - "runtime_checkable": ("litellm.files.main", "runtime_checkable"), - "rust": ("litellm.rust_bridge", "rust"), - "safe_deep_copy": ("litellm.litellm_core_utils.core_helpers", "safe_deep_copy"), - "sagemaker_chat_completion": ("litellm.main", "sagemaker_chat_completion"), - "sagemaker_llm": ("litellm.main", "sagemaker_llm"), - "sanitize_tool_use_ids_in_anthropic_messages": ( - "litellm.llms.anthropic.common_utils", - "sanitize_tool_use_ids_in_anthropic_messages", - ), - "sap_gen_ai_hub_chat_completions": ("litellm.main", "sap_gen_ai_hub_chat_completions"), - "sap_gen_ai_hub_emb": ("litellm.main", "sap_gen_ai_hub_emb"), - "search": ("litellm.search.main", "search"), - "should_run_mock_completion": ("litellm.utils", "should_run_mock_completion"), - "speech": ("litellm.main", "speech"), - "stream_chunk_builder": ("litellm.main", "stream_chunk_builder"), - "stream_chunk_builder_text_completion": ("litellm.main", "stream_chunk_builder_text_completion"), - "stringify_json_tool_call_content": ( - "litellm.litellm_core_utils.prompt_templates.factory", - "stringify_json_tool_call_content", - ), - "strip_empty_content_blocks_from_anthropic_messages": ( - "litellm.llms.anthropic.common_utils", - "strip_empty_content_blocks_from_anthropic_messages", - ), - "strip_reasoning_summary_aliases_from_optional_params": ( - "litellm.utils", - "strip_reasoning_summary_aliases_from_optional_params", - ), - "supports_httpx_timeout": ("litellm.utils", "supports_httpx_timeout"), - "text_completion": ("litellm.main", "text_completion"), - "together_rerank": ("litellm.rerank_api.main", "together_rerank"), - "tracer": ("litellm.litellm_core_utils.dd_tracing", "tracer"), - "transcription": ("litellm.main", "transcription"), - "updateDeployment": ("litellm.types.router", "updateDeployment"), - "updateLiteLLMParams": ("litellm.types.router", "updateLiteLLMParams"), - "update_cache": ("litellm.caching.caching", "update_cache"), - "update_messages_with_model_file_ids": ( - "litellm.litellm_core_utils.prompt_templates.common_utils", - "update_messages_with_model_file_ids", - ), - "update_responses_input_with_model_file_ids": ( - "litellm.litellm_core_utils.prompt_templates.common_utils", - "update_responses_input_with_model_file_ids", - ), - "update_responses_tools_with_model_file_ids": ( - "litellm.litellm_core_utils.prompt_templates.common_utils", - "update_responses_tools_with_model_file_ids", - ), - "upload_container_file": ("litellm.containers.main", "upload_container_file"), - "urlsplit": ("litellm.main", "urlsplit"), - "validate_and_fix_openai_messages": ("litellm.utils", "validate_and_fix_openai_messages"), - "validate_and_fix_openai_tools": ("litellm.utils", "validate_and_fix_openai_tools"), - "validate_and_fix_thinking_param": ("litellm.utils", "validate_and_fix_thinking_param"), - "validate_anthropic_api_metadata": ( - "litellm.llms.anthropic.experimental_pass_through.messages.handler", - "validate_anthropic_api_metadata", - ), - "validate_chat_completion_tool_choice": ("litellm.utils", "validate_chat_completion_tool_choice"), - "validate_openai_optional_params": ("litellm.utils", "validate_openai_optional_params"), - "vector_store_file_content": ("litellm.vector_store_files.main", "retrieve_content"), - "vector_store_file_create": ("litellm.vector_store_files.main", "create"), - "vector_store_file_delete": ("litellm.vector_store_files.main", "delete"), - "vector_store_file_list": ("litellm.vector_store_files.main", "list"), - "vector_store_file_retrieve": ("litellm.vector_store_files.main", "retrieve"), - "vector_store_file_update": ("litellm.vector_store_files.main", "update"), - "vertex_ai_batches_instance": ("litellm.batches.main", "vertex_ai_batches_instance"), - "vertex_ai_files_instance": ("litellm.files.main", "vertex_ai_files_instance"), - "vertex_chat_completion": ("litellm.main", "vertex_chat_completion"), - "vertex_embedding": ("litellm.main", "vertex_embedding"), - "vertex_fine_tuning_apis_instance": ("litellm.fine_tuning.main", "vertex_fine_tuning_apis_instance"), - "vertex_gemma_chat_completion": ("litellm.main", "vertex_gemma_chat_completion"), - "vertex_image_generation": ("litellm.main", "vertex_image_generation"), - "vertex_model_garden_chat_completion": ("litellm.main", "vertex_model_garden_chat_completion"), - "vertex_multimodal_embedding": ("litellm.main", "vertex_multimodal_embedding"), - "vertex_partner_models_chat_completion": ("litellm.main", "vertex_partner_models_chat_completion"), - "video_content": ("litellm.videos.main", "video_content"), - "video_create_character": ("litellm.videos.main", "video_create_character"), - "video_edit": ("litellm.videos.main", "video_edit"), - "video_extension": ("litellm.videos.main", "video_extension"), - "video_generation": ("litellm.videos.main", "video_generation"), - "video_get_character": ("litellm.videos.main", "video_get_character"), - "video_list": ("litellm.videos.main", "video_list"), - "video_remix": ("litellm.videos.main", "video_remix"), - "video_status": ("litellm.videos.main", "video_status"), - "wait": ("litellm.batch_completion.main", "wait"), - "watsonx_chat_completion": ("litellm.main", "watsonx_chat_completion"), - } -) - -_SDK_MODULE_ALIASES: Final[Mapping[str, str]] = MappingProxyType( - { - "additional_logging_utils": "litellm.integrations.additional_logging_utils", - "agentops": "litellm.integrations.agentops", - "aleph_alpha": "litellm.llms.deprecated_providers.aleph_alpha", - "anthropic_cache_control_hook": "litellm.integrations.anthropic_cache_control_hook", - "argilla": "litellm.integrations.argilla", - "arize": "litellm.integrations.arize", - "asyncio": "asyncio", - "athina": "litellm.integrations.athina", - "azure_sentinel": "litellm.integrations.azure_sentinel", - "azure_storage": "litellm.integrations.azure_storage", - "base64": "base64", - "cohere_embed": "litellm.llms.cohere.embed.handler", - "contextvars": "contextvars", - "custom_batch_logger": "litellm.integrations.custom_batch_logger", - "custom_guardrail": "litellm.integrations.custom_guardrail", - "custom_logger": "litellm.integrations.custom_logger", - "custom_prompt_management": "litellm.integrations.custom_prompt_management", - "datadog": "litellm.integrations.datadog", - "datetime": "datetime", - "deepeval": "litellm.integrations.deepeval", - "dotenv": "dotenv", - "dotprompt": "litellm.integrations.dotprompt", - "dynamodb": "litellm.integrations.dynamodb", - "email_templates": "litellm.integrations.email_templates", - "enum": "enum", - "futures": "concurrent.futures", - "galileo": "litellm.integrations.galileo", - "gcs_bucket": "litellm.integrations.gcs_bucket", - "gcs_pubsub": "litellm.integrations.gcs_pubsub", - "generic_api": "litellm.integrations.generic_api", - "greenscale": "litellm.integrations.greenscale", - "heapq": "heapq", - "helicone": "litellm.integrations.helicone", - "helicone_mock_client": "litellm.integrations.helicone_mock_client", - "humanloop": "litellm.integrations.humanloop", - "importlib": "importlib", - "inspect": "inspect", - "json": "json", - "lago": "litellm.integrations.lago", - "langfuse": "litellm.integrations.langfuse", - "langsmith": "litellm.integrations.langsmith", - "langsmith_mock_client": "litellm.integrations.langsmith_mock_client", - "litellm": "litellm", - "litellm_agent": "litellm.integrations.litellm_agent", - "literal_ai": "litellm.integrations.literal_ai", - "logfire_logger": "litellm.integrations.logfire_logger", - "lunary": "litellm.integrations.lunary", - "mimetypes": "mimetypes", - "mlflow": "litellm.integrations.mlflow", - "mock_client_factory": "litellm.integrations.mock_client_factory", - "newrelic": "litellm.integrations.newrelic", - "ollama": "litellm.llms.ollama.completion.handler", - "oobabooga": "litellm.llms.oobabooga.chat.oobabooga", - "openai": "openai", - "openmeter": "litellm.integrations.openmeter", - "opentelemetry": "litellm.integrations.opentelemetry", - "opentelemetry_utils": "litellm.integrations.opentelemetry_utils", - "opik": "litellm.integrations.opik", - "otel": "litellm.integrations.otel", - "palm": "litellm.llms.deprecated_providers.palm", - "petals_handler": "litellm.llms.petals.completion.handler", - "posthog": "litellm.integrations.posthog", - "posthog_mock_client": "litellm.integrations.posthog_mock_client", - "prompt_layer": "litellm.integrations.prompt_layer", - "prompt_management_base": "litellm.integrations.prompt_management_base", - "random": "random", - "rust_ocr_bridge": "litellm.rust_bridge.ocr", - "s3": "litellm.integrations.s3", - "s3_v2": "litellm.integrations.s3_v2", - "sqs": "litellm.integrations.sqs", - "supabase": "litellm.integrations.supabase", - "sys": "sys", - "tiktoken": "tiktoken", - "time": "time", - "traceback": "traceback", - "traceloop": "litellm.integrations.traceloop", - "uuid": "fastuuid", - "uuid_module": "uuid", - "vertex_ai_non_gemini": "litellm.llms.vertex_ai.vertex_ai_non_gemini", - "vllm_handler": "litellm.llms.vllm.completion.handler", - "anthropic": "litellm.anthropic_interface", - "httpx": "httpx", - "interactions": "litellm.interactions", - "rag": "litellm.rag", - } -) - # Export all name tuples and import maps for use in _lazy_imports.py __all__ = [ "BEDROCK_TYPES_NAMES", @@ -2657,7 +1490,6 @@ __all__ = [ "LLM_CLIENT_CACHE_NAMES", "LLM_CONFIG_NAMES", "LLM_PROVIDER_LOGIC_NAMES", - "STAR_IMPORT_PUBLIC_NAMES", "TOKEN_COUNTER_NAMES", "TYPES_NAMES", "TYPES_UTILS_NAMES", @@ -2670,1534 +1502,9 @@ __all__ = [ "_LITELLM_LOGGING_IMPORT_MAP", "_LLM_CONFIGS_IMPORT_MAP", "_LLM_PROVIDER_LOGIC_IMPORT_MAP", - "_SDK_MODULE_ALIASES", - "_SDK_SYMBOLS_IMPORT_MAP", "_TOKEN_COUNTER_IMPORT_MAP", "_TYPES_IMPORT_MAP", "_TYPES_UTILS_IMPORT_MAP", "_UTILS_IMPORT_MAP", "_UTILS_MODULE_IMPORT_MAP", ] - - -STAR_IMPORT_PUBLIC_NAMES: Final = ( - "AI21ChatConfig", - "AI21Config", - "ALL_RESPONSES_API_TOOL_PARAMS", - "APIConnectionError", - "APIError", - "APIResponseValidationError", - "AZURE_DEFAULT_API_VERSION", - "AZURE_OPENAI_AUDIO_PROVIDERS", - "AdapterCompletionStreamWrapper", - "AdapterItem", - "AdaptiveRouterConfig", - "AdaptiveRouterPreferences", - "AdaptiveRouterWeights", - "AlephAlphaConfig", - "AlertingConfig", - "AllEmbeddingInputValues", - "AllMessageValues", - "AllPromptValues", - "AllowedFailsPolicy", - "AmazonTitanV2Config", - "Annotated", - "AnthropicBatchesHandler", - "AnthropicChatCompletion", - "AnthropicMessagesRequestUtils", - "AnthropicMessagesResponse", - "AnthropicMetadata", - "AnthropicModelInfo", - "AnthropicThinkingParam", - "Any", - "Assistant", - "AssistantDeleted", - "AssistantEventHandler", - "AssistantStreamManager", - "AssistantToolParam", - "AssistantsTypedDict", - "AsyncAssistantEventHandler", - "AsyncAssistantStreamManager", - "AsyncCompletions", - "AsyncCursorPage", - "AsyncHTTPHandler", - "AsyncIterator", - "AsyncOpenAI", - "Attachment", - "AttachmentTool", - "AuthenticationError", - "AutoRouterCapabilityLimit", - "AzureAIEmbedding", - "AzureAnthropicChatCompletion", - "AzureAssistantsAPI", - "AzureAudioTranscription", - "AzureBatchesAPI", - "AzureChatCompletion", - "AzureOpenAIFilesAPI", - "AzureOpenAIFineTuningAPI", - "AzureOpenAIO1ChatCompletion", - "AzureTextCompletion", - "BATCH_GUARDRAIL_RESPONSE_FIELD", - "BEDROCK_CONVERSE_MODELS", - "BEDROCK_EMBEDDING_PROVIDERS_LITERAL", - "BEDROCK_INVOKE_PROVIDERS_LITERAL", - "BadGatewayError", - "BadRequestError", - "BaseAnthropicMessagesConfig", - "BaseConfig", - "BaseImageEditConfig", - "BaseImageGenerationConfig", - "BaseLLMAIOHTTPHandler", - "BaseLLMException", - "BaseLLMHTTPHandler", - "BaseLiteLLMOpenAIResponseObject", - "BaseModel", - "BaseOCRConfig", - "BaseRerankConfig", - "BaseResponsesAPIConfig", - "BaseResponsesAPIStreamingIterator", - "BaseSearchConfig", - "BaseVideoConfig", - "Batch", - "BatchGuardrailRecord", - "BatchGuardrailReport", - "BatchJobStatus", - "BatchRequestCounts", - "BedrockBatchesHandler", - "BedrockConverseLLM", - "BedrockEmbedding", - "BedrockFilesHandler", - "BedrockImageEdit", - "BedrockImageGeneration", - "BedrockModelInfo", - "BedrockRerankHandler", - "BudgetExceededError", - "BudgetManager", - "BytezChatConfig", - "CALLBACK_TYPES", - "CARRY_UNMATCHED_MESSAGE_POINTS", - "COHERE_DEFAULT_EMBEDDING_INPUT_TYPE", - "CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS", - "CREATE_FILE_REQUESTS_PURPOSE", - "CallTypes", - "Callable", - "CancelBatchRequest", - "CharacterObject", - "Chat", - "ChatCompletionAnnotation", - "ChatCompletionAnnotationURLCitation", - "ChatCompletionAssistantContentValue", - "ChatCompletionAssistantMessage", - "ChatCompletionAssistantToolCall", - "ChatCompletionAudioDelta", - "ChatCompletionAudioObject", - "ChatCompletionAudioParam", - "ChatCompletionCachedContent", - "ChatCompletionChunk", - "ChatCompletionContentPartInputAudioParam", - "ChatCompletionDeltaChunk", - "ChatCompletionDeltaToolCallChunk", - "ChatCompletionDeveloperMessage", - "ChatCompletionDocumentObject", - "ChatCompletionFileObject", - "ChatCompletionFileObjectFile", - "ChatCompletionFunctionMessage", - "ChatCompletionImageObject", - "ChatCompletionImageUrlObject", - "ChatCompletionMessageToolCall", - "ChatCompletionModality", - "ChatCompletionNamedToolChoiceParam", - "ChatCompletionPredictionContentParam", - "ChatCompletionReasoningItem", - "ChatCompletionReasoningSummaryTextBlock", - "ChatCompletionRedactedThinkingBlock", - "ChatCompletionRequest", - "ChatCompletionResponseMessage", - "ChatCompletionSystemMessage", - "ChatCompletionTextObject", - "ChatCompletionThinkingBlock", - "ChatCompletionToolCallChunk", - "ChatCompletionToolCallFunctionChunk", - "ChatCompletionToolChoiceFunctionParam", - "ChatCompletionToolChoiceObjectParam", - "ChatCompletionToolChoiceStringValues", - "ChatCompletionToolChoiceValues", - "ChatCompletionToolMessage", - "ChatCompletionToolParam", - "ChatCompletionToolParamFunctionChunk", - "ChatCompletionToolReferenceObject", - "ChatCompletionUsageBlock", - "ChatCompletionUserMessage", - "ChatCompletionVideoObject", - "ChatCompletionVideoUrlObject", - "Choices", - "ChunkProcessor", - "CitationsObject", - "ClarifaiConfig", - "ClassVar", - "ClassifierPlugin", - "CodeInterpreterToolParam", - "CodestralTextCompletion", - "CohereModelInfo", - "CompletionRequest", - "CompletionTimeout", - "CompletionTokensDetails", - "Completions", - "ComputerToolParam", - "ConfigDict", - "ConfigurableClientsideParamsCustomAuth", - "ConsumedRequestTagsStamp", - "ContentPartAddedEvent", - "ContentPartDoneEvent", - "ContentPartDonePartOutputText", - "ContentPartDonePartReasoningText", - "ContentPartDonePartRefusal", - "ContentPolicyViolationError", - "ContextManagementEntry", - "ContextWindowExceededError", - "Coroutine", - "CreateBatchRequest", - "CreateFileRequest", - "CreateVideoRequest", - "CredentialLiteLLMParams", - "CustomLLM", - "CustomLLMItem", - "CustomLogger", - "CustomPricingLiteLLMParams", - "CustomRoutingStrategyBase", - "CustomStreamWrapper", - "CustomToolCallOutputItem", - "DEFAULT_ALLOWED_FAILS", - "DEFAULT_BATCH_SIZE", - "DEFAULT_FLUSH_INTERVAL_SECONDS", - "DEFAULT_IMAGE_ENDPOINT_MODEL", - "DEFAULT_IN_MEMORY_TTL", - "DEFAULT_MAX_RETRIES", - "DEFAULT_MAX_TOKENS", - "DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", - "DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT", - "DEFAULT_POLLING_INTERVAL", - "DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", - "DEFAULT_REPLICATE_POLLING_RETRIES", - "DEFAULT_REQUEST_TIMEOUT", - "DEFAULT_SOFT_BUDGET", - "DEFAULT_VIDEO_ENDPOINT_MODEL", - "DatabricksEmbeddingHandler", - "DatadogInitParams", - "DecodedResponseId", - "DeleteResponseResult", - "Deployment", - "DeploymentTypedDict", - "Dict", - "Discriminator", - "DocumentObject", - "DualCache", - "EmbeddingCreateParams", - "EmbeddingInput", - "EmbeddingRequest", - "EmbeddingResponse", - "Enum", - "ErrorEvent", - "ErrorEventError", - "FIRST_COMPLETED", - "FORWARDED_KWARGS_KEYS", - "FallbackAccessCheck", - "Field", - "FileContent", - "FileContentProvider", - "FileContentRequest", - "FileContentStreamingResponse", - "FileContentStreamingResult", - "FileCreateProvider", - "FileDeleteProvider", - "FileDeleted", - "FileExpiresAfter", - "FileListPage", - "FileListProvider", - "FileObject", - "FileRetrieveProvider", - "FileSearchCallCompletedEvent", - "FileSearchCallInProgressEvent", - "FileSearchCallSearchingEvent", - "FileSearchTool", - "FileSearchToolParam", - "FileTypes", - "Final", - "FineTuningConfig", - "FineTuningJob", - "FineTuningJobCreate", - "FlowItem", - "Function", - "FunctionCallArgumentsDeltaEvent", - "FunctionCallArgumentsDoneEvent", - "GDCGeminiConfig", - "GeminiModelInfo", - "GenAIHubOrchestration", - "Generator", - "Generic", - "GenericBudgetWindowDetails", - "GenericChatCompletionMessage", - "GenericEvent", - "GenericLiteLLMParams", - "GenericResponseOutputItem", - "GenericResponseOutputItemContentAnnotation", - "GoogleBatchEmbeddings", - "GroqChatCompletion", - "GuardrailLiteLLMParams", - "GuardrailTypedDict", - "HTTPHandler", - "HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", - "HerokuChatConfig", - "HiddenParams", - "HttpxBinaryResponseContent", - "HuggingFaceEmbedding", - "Hyperparameters", - "IBMWatsonXMixin", - "IO", - "IOBase", - "ImageEditOptionalRequestParams", - "ImageFetchError", - "ImageFileObject", - "ImageGenerationPartialImageEvent", - "ImageGenerationRequestQuality", - "ImageResponse", - "ImageURLListItem", - "ImageURLObject", - "IncompleteDetails", - "InputTokensDetails", - "InternalServerError", - "InvalidRequestError", - "Iterable", - "Iterator", - "JSONProviderRegistry", - "JSONSchemaValidationError", - "KeyManagementSettings", - "LIST_BATCHES_SUPPORTED_PROVIDERS", - "LITELLM_CHAT_PROVIDERS", - "LITELLM_EXCEPTION_TYPES", - "LITELLM_IMAGE_VARIATION_PROVIDERS", - "LemonadeChatConfig", - "List", - "ListBatchRequest", - "ListBatchesSupportedProvider", - "LiteLLM", - "LiteLLMBatch", - "LiteLLMBatchCreateRequest", - "LiteLLMCompletionTransformationHandler", - "LiteLLMFineTuningJob", - "LiteLLMFineTuningJobCreate", - "LiteLLMLoggingObj", - "LiteLLMMessagesToCompletionTransformationHandler", - "LiteLLMMessagesToResponsesAPIHandler", - "LiteLLMParamsTypedDict", - "LiteLLMResponsesTransformationHandler", - "LiteLLMUnknownProvider", - "LiteLLM_Params", - "LiteLLM_RouterFileObject", - "Literal", - "LlmProviders", - "Logging", - "MCPCallArgumentsDeltaEvent", - "MCPCallArgumentsDoneEvent", - "MCPCallCompletedEvent", - "MCPCallFailedEvent", - "MCPCallInProgressEvent", - "MCPListToolsCompletedEvent", - "MCPListToolsFailedEvent", - "MCPListToolsInProgressEvent", - "MCPTool", - "MOCK_RESPONSE_TYPE", - "Mapping", - "MappingProxyType", - "Message", - "MessageContent", - "MessageContentImageFileObject", - "MessageContentImageURLObject", - "MessageContentTextObject", - "MessageData", - "MirroredPricingParams", - "MockException", - "MockRouterTestingParams", - "ModelConfig", - "ModelGroupInfo", - "ModelGroupSettings", - "ModelInfo", - "ModelResponse", - "ModelResponseStream", - "MyLocal", - "NOT_GIVEN", - "NewRelicInitParams", - "NonNegativeInt", - "NotFoundError", - "NotGiven", - "NotRequired", - "NvidiaRivaAudioTranscription", - "NvidiaRivaAudioTranscriptionConfig", - "OCIChatConfig", - "OCRResponse", - "OCR_REQUEST_FORMAT_PARAM", - "OPENAI_CHAT_COMPLETION_PARAMS", - "OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS", - "OPENAI_FINISH_REASONS", - "OPTIONAL_KWARGS_KEYS", - "OVHCloudChatConfig", - "Omit", - "OpenAI", - "OpenAIAssistantsAPI", - "OpenAIAudioTranscription", - "OpenAIAudioTranscriptionOptionalParams", - "OpenAIBatchResponse", - "OpenAIBatchResult", - "OpenAIBatchesAPI", - "OpenAIChatCompletion", - "OpenAIChatCompletionAssistantMessage", - "OpenAIChatCompletionChoices", - "OpenAIChatCompletionChunk", - "OpenAIChatCompletionDeveloperMessage", - "OpenAIChatCompletionFinishReason", - "OpenAIChatCompletionLogprobs", - "OpenAIChatCompletionLogprobsContent", - "OpenAIChatCompletionLogprobsContentTopLogprobs", - "OpenAIChatCompletionResponse", - "OpenAIChatCompletionSystemMessage", - "OpenAIChatCompletionTextObject", - "OpenAIChatCompletionToolParam", - "OpenAIChatCompletionUserMessage", - "OpenAICreateFileRequestOptionalParams", - "OpenAICreateThreadParamsMessage", - "OpenAICreateThreadParamsToolResources", - "OpenAIEmbedding", - "OpenAIError", - "OpenAIErrorBody", - "OpenAIFileObject", - "OpenAIFilesAPI", - "OpenAIFilesPurpose", - "OpenAIFineTuningAPI", - "OpenAIGPT5Config", - "OpenAIImageEditOptionalParams", - "OpenAIImageGenerationOptionalParams", - "OpenAIImageVariationOptionalParams", - "OpenAIImageVariationsHandler", - "OpenAILikeChatHandler", - "OpenAILikeEmbeddingHandler", - "OpenAILikeResponsesConfig", - "OpenAIMcpServerTool", - "OpenAIMessage", - "OpenAIMessageContent", - "OpenAIMessageContentListBlock", - "OpenAIModerationResponse", - "OpenAIModerationResult", - "OpenAIRealtimeContentPartDone", - "OpenAIRealtimeConversationCreated", - "OpenAIRealtimeConversationItemAdded", - "OpenAIRealtimeConversationItemCreated", - "OpenAIRealtimeConversationItemDone", - "OpenAIRealtimeConversationObject", - "OpenAIRealtimeDoneEvent", - "OpenAIRealtimeEventTypes", - "OpenAIRealtimeEvents", - "OpenAIRealtimeFunctionCallArgumentsDone", - "OpenAIRealtimeInputAudioBufferSpeechEvent", - "OpenAIRealtimeInputAudioTranscriptionCompleted", - "OpenAIRealtimeInputAudioTranscriptionDelta", - "OpenAIRealtimeOutputItemDone", - "OpenAIRealtimeResponseAudioDone", - "OpenAIRealtimeResponseContentPart", - "OpenAIRealtimeResponseContentPartAdded", - "OpenAIRealtimeResponseDelta", - "OpenAIRealtimeResponseDoneObject", - "OpenAIRealtimeResponseTextDone", - "OpenAIRealtimeResponseUsage", - "OpenAIRealtimeStreamList", - "OpenAIRealtimeStreamResponseBaseObject", - "OpenAIRealtimeStreamResponseOutputItem", - "OpenAIRealtimeStreamResponseOutputItemAdded", - "OpenAIRealtimeStreamResponseOutputItemContent", - "OpenAIRealtimeStreamSession", - "OpenAIRealtimeStreamSessionEvents", - "OpenAIRealtimeTurnDetection", - "OpenAIRealtimeUsageTokenDetails", - "OpenAITextCompletion", - "OpenAITextCompletionUserMessage", - "OpenAIVideoObject", - "OpenAIWebSearchOptions", - "OpenAIWebSearchUserLocation", - "OpenAIWebSearchUserLocationApproximate", - "Optional", - "OptionalPreCallChecks", - "OutputCodeInterpreterCall", - "OutputCodeInterpreterCallLog", - "OutputFunctionToolCall", - "OutputImageGenerationCall", - "OutputItemAddedEvent", - "OutputItemDoneEvent", - "OutputText", - "OutputTextAnnotationAddedEvent", - "OutputTextDeltaEvent", - "OutputTextDoneEvent", - "OutputTokensDetails", - "PART_UNION_TYPES", - "PalmConfig", - "PathLike", - "PermissionDeniedError", - "Phase", - "PreRoutingHookResponse", - "PreRoutingStrategy", - "PredibaseChatCompletion", - "PrivateAttr", - "PromptCacheBreakpoint", - "PromptCacheOptions", - "PromptObject", - "PromptSpec", - "PromptTokensDetails", - "Protocol", - "ProviderConfigManager", - "ProviderSpecificHeader", - "ProviderSpecificHeaderUtils", - "REASONING_EFFORT", - "REPEATED_STREAMING_CHUNK_LIMIT", - "ROUTER_MAX_FALLBACKS", - "RateLimitError", - "RateLimitErrorCategory", - "RateLimitType", - "RawRequestTypedDict", - "ReadOnly", - "Reasoning", - "ReasoningSummaryPartDoneEvent", - "ReasoningSummaryTextDeltaEvent", - "ReasoningSummaryTextDoneEvent", - "RedisCache", - "RefusalDeltaEvent", - "RefusalDoneEvent", - "RequestType", - "Required", - "RerankResponse", - "Response", - "ResponseAPIUsage", - "ResponseCompletedEvent", - "ResponseCreatedEvent", - "ResponseFailedEvent", - "ResponseFunctionToolCall", - "ResponseInProgressEvent", - "ResponseIncludable", - "ResponseIncompleteEvent", - "ResponseInputParam", - "ResponseOutputItem", - "ResponsePartAddedEvent", - "ResponseText", - "ResponsesAPIOptionalRequestParams", - "ResponsesAPIRequestParams", - "ResponsesAPIRequestUtils", - "ResponsesAPIResponse", - "ResponsesAPIStatus", - "ResponsesAPIStreamEvents", - "ResponsesAPIStreamOptions", - "ResponsesAPIStreamingResponse", - "ResponsesToolUsage", - "RetrieveBatchRequest", - "RetryPolicy", - "Router", - "RouterCacheEnum", - "RouterConfig", - "RouterErrors", - "RouterGeneralSettings", - "RouterModelGroupAliasItem", - "RouterRateLimitError", - "RouterRateLimitErrorBasic", - "RoutingContext", - "RoutingGroup", - "RoutingPlugin", - "RoutingStrategy", - "Run", - "SPECIAL_MODEL_INFO_PARAMS", - "SagemakerChatHandler", - "SagemakerLLM", - "Scheduler", - "SchedulerCacheKeys", - "SearchProvider", - "SearchProviders", - "SearchResponse", - "SearchToolInfoTypedDict", - "SearchToolLiteLLMParams", - "SearchToolTypedDict", - "Sequence", - "SerializerFunctionWrapHandler", - "ServiceUnavailableError", - "Set", - "ShellToolParam", - "SlackAlerting", - "StandardLoggingRoutingDecision", - "StreamingChoices", - "SyncCursorPage", - "TYPE_CHECKING", - "TaggedPreRoutingStrategy", - "TextChoices", - "TextCompletionResponse", - "TextCompletionStreamWrapper", - "Thread", - "ThreadPoolExecutor", - "Timeout", - "TogetherAIRerank", - "Tool", - "ToolChoice", - "ToolMessageContentPart", - "ToolParam", - "ToolResourcesCodeInterpreter", - "ToolResourcesFileSearch", - "ToolResourcesFileSearchVectorStore", - "TopazModelInfo", - "TranscriptionResponse", - "Tuple", - "Type", - "TypeAlias", - "TypeVar", - "TypedDict", - "Union", - "UnprocessableEntityError", - "UnsupportedParamsError", - "UpdateRouterConfig", - "Usage", - "VALID_LITELLM_ENVIRONMENTS", - "ValidAssistantMessageContentTypes", - "ValidAssistantMessageContentTypesLiteral", - "ValidChatCompletionMessageContentTypes", - "ValidChatCompletionMessageContentTypesLiteral", - "ValidUserMessageContentTypes", - "ValidUserMessageContentTypesLiteral", - "VectorStoreIndexRegistry", - "VectorStoreRegistry", - "VertexAIBatchPrediction", - "VertexAIFilesHandler", - "VertexAIGemmaModels", - "VertexAIModelGardenModels", - "VertexAIModelRoute", - "VertexAIPartnerModels", - "VertexAITextEmbeddingConfig", - "VertexEmbedding", - "VertexFineTuningAPI", - "VertexImageGeneration", - "VertexLLM", - "VertexMultimodalEmbedding", - "VideoCreateOptionalRequestParams", - "VideoGenerationRequestUtils", - "VideoObject", - "WANDB_MODELS", - "WATSONX_DEFAULT_API_VERSION", - "WatsonXChatHandler", - "WebSearchCallCompletedEvent", - "WebSearchCallInProgressEvent", - "WebSearchCallSearchingEvent", - "WebSearchOptions", - "WebSearchOptionsUserLocation", - "WebSearchOptionsUserLocationApproximate", - "WebSearchToolUsage", - "XAIModelInfo", - "a_add_message", - "aadapter_completion", - "aadapter_generate_content", - "acancel_batch", - "acancel_eval", - "acancel_fine_tuning_job", - "acancel_responses", - "acancel_run", - "aclient_session", - "acode_interpreter_tool", - "acompact_responses", - "acompletion", - "acompletion_with_retries", - "acount_tokens", - "acreate_agent", - "acreate_assistants", - "acreate_batch", - "acreate_container", - "acreate_eval", - "acreate_file", - "acreate_fine_tuning_job", - "acreate_realtime_client_secret", - "acreate_realtime_transcription_session", - "acreate_run", - "acreate_sandbox", - "acreate_skill", - "acreate_thread", - "adapter_completion", - "adapters", - "add_function_to_prompt", - "add_known_models", - "add_message", - "add_provider_specific_params_to_optional_params", - "add_system_prompt_to_messages", - "add_trusted_model_credentials_to_litellm_params", - "add_user_information_to_llm_headers", - "additional_logging_utils", - "adelete_agent", - "adelete_assistant", - "adelete_container", - "adelete_eval", - "adelete_responses", - "adelete_run", - "adelete_sandbox", - "adelete_skill", - "aembedding", - "afile_content", - "afile_delete", - "afile_list", - "afile_retrieve", - "agenerate_content", - "agent_search_embedding_model", - "agentops", - "aget_agent", - "aget_assistants", - "aget_eval", - "aget_messages", - "aget_responses", - "aget_run", - "aget_skill", - "aget_thread", - "ahealth_check", - "ai21_chat_models", - "ai21_key", - "ai21_models", - "aimage_edit", - "aimage_generation", - "aimage_variation", - "aiml_models", - "aingest", - "aiohttp_trust_env", - "aleph_alpha", - "aleph_alpha_key", - "aleph_alpha_models", - "alist_agent_versions", - "alist_agents", - "alist_batches", - "alist_container_files", - "alist_containers", - "alist_evals", - "alist_fine_tuning_jobs", - "alist_input_items", - "alist_runs", - "alist_skills", - "all_embedding_models", - "all_litellm_params", - "allm_passthrough_route", - "allow_dynamic_callback_disabling", - "allowed_fails", - "amazon_nova_api_key", - "amazon_nova_models", - "amoderation", - "annotations", - "anthropic", - "anthropic_batches_instance", - "anthropic_beta_headers_manager", - "anthropic_beta_headers_url", - "anthropic_cache_control_hook", - "anthropic_chat_completions", - "anthropic_interface", - "anthropic_key", - "anthropic_messages", - "anthropic_messages_handler", - "anthropic_models", - "anthropic_prompt_caching_ttl", - "anthropic_sse_ping_interval_seconds", - "anyscale_models", - "aocr", - "api_base", - "api_key", - "api_version", - "aquery", - "arealtime_calls", - "arerank", - "aresponses", - "aresponses_api_with_mcp", - "aresponses_with_retries", - "aretrieve_batch", - "aretrieve_container", - "aretrieve_fine_tuning_job", - "argilla", - "argilla_batch_size", - "argilla_transformation_object", - "arize", - "arun_code", - "arun_thread", - "arun_thread_stream", - "asearch", - "aspeech", - "assemblyai_models", - "assistants", - "async_completion_with_fallbacks", - "async_mock_completion_streaming_obj", - "asyncio", - "atext_completion", - "athina", - "atranscription", - "audit_log_callbacks", - "aupload_container_file", - "autorouter_presets_url", - "avector_store_file_content", - "avector_store_file_create", - "avector_store_file_delete", - "avector_store_file_list", - "avector_store_file_retrieve", - "avector_store_file_update", - "avideo_content", - "avideo_create_character", - "avideo_edit", - "avideo_extension", - "avideo_generation", - "avideo_get_character", - "avideo_list", - "avideo_remix", - "avideo_status", - "aws_polly_models", - "aws_sqs_callback_params", - "azure_ai_embedding", - "azure_ai_models", - "azure_anthropic_chat_completions", - "azure_anthropic_models", - "azure_assistants_api", - "azure_audio_transcriptions", - "azure_batches_instance", - "azure_chat_completions", - "azure_embedding_models", - "azure_files_instance", - "azure_fine_tuning_apis_instance", - "azure_key", - "azure_llms", - "azure_models", - "azure_o1_chat_completions", - "azure_sentinel", - "azure_storage", - "azure_text_completions", - "azure_text_models", - "banned_keywords_list", - "base64", - "base_llm_aiohttp_handler", - "base_llm_http_handler", - "baseten_key", - "baseten_models", - "batch_completion", - "batch_completion_models", - "batch_completion_models_all_responses", - "batches", - "bedrock_converse_chat_completion", - "bedrock_converse_models", - "bedrock_embedding", - "bedrock_embedding_models", - "bedrock_files_instance", - "bedrock_image_edit", - "bedrock_image_generation", - "bedrock_mantle_models", - "bedrock_models", - "bedrock_request_metadata_fields", - "bedrock_rerank", - "bfl_image_edit", - "bfl_image_generation", - "black_forest_labs_models", - "block_requests_for_models_without_pricing", - "blocked_user_list", - "blog_posts_url", - "budget_duration", - "budget_exceeded_throttle_percentage", - "budget_manager", - "budget_rollover", - "build_code_interpreter_log_outputs", - "bytez_key", - "bytez_transformation", - "cache", - "caching", - "caching_with_models", - "calculate_request_duration", - "callback_settings", - "callbacks", - "cancel_batch", - "cancel_eval", - "cancel_fine_tuning_job", - "cancel_responses", - "cancel_run", - "cast", - "cerebras_models", - "chatgpt_models", - "check_provider_endpoint", - "clarifai_key", - "clarifai_models", - "client", - "client_session", - "close_litellm_async_clients", - "cloudflare_api_key", - "cloudflare_models", - "codestral_models", - "codestral_text_completions", - "cohere_chat_models", - "cohere_embed", - "cohere_embedding_models", - "cohere_key", - "cohere_models", - "cold_storage_custom_logger", - "cometapi_key", - "cometapi_models", - "common_cloud_provider_auth_params", - "compact_responses", - "completion", - "completion_extras", - "completion_with_fallbacks", - "completion_with_retries", - "compress", - "compression", - "config_completion", - "config_path", - "constants", - "containers", - "content_policy_fallbacks", - "context_window_fallbacks", - "contextmanager", - "contextvars", - "convert_file_document_to_url_document", - "convert_model_response_to_streaming", - "convert_to_model_response_object", - "cost_calculator", - "cost_discount_config", - "cost_margin_config", - "create_agent", - "create_assistants", - "create_batch", - "create_container", - "create_eval", - "create_file", - "create_fine_tuning_job", - "create_pretrained_tokenizer", - "create_run", - "create_skill", - "create_thread", - "create_tokenizer", - "credential_list", - "custom_batch_logger", - "custom_chat_llm_router", - "custom_guardrail", - "custom_logger", - "custom_prometheus_metadata_labels", - "custom_prometheus_tags", - "custom_prompt", - "custom_prompt_dict", - "custom_prompt_management", - "custom_provider_map", - "darkbloom_models", - "dashscope_models", - "databricks_embedding", - "databricks_key", - "databricks_models", - "dataclass", - "datadog", - "datadog_llm_observability_params", - "datadog_params", - "datadog_use_v1", - "datarobot_key", - "datarobot_models", - "datetime", - "decode_video_id_with_provider", - "declared_authenticating_provider", - "deepcopy", - "deepeval", - "deepgram_models", - "deepinfra_models", - "deepseek_models", - "default_fallbacks", - "default_in_memory_ttl", - "default_internal_user_params", - "default_key_generate_params", - "default_key_max_budget_alert_emails", - "default_max_internal_user_budget", - "default_redis_batch_cache_expiry", - "default_redis_ttl", - "default_soft_budget", - "default_team_params", - "default_team_settings", - "delete_agent", - "delete_assistant", - "delete_container", - "delete_eval", - "delete_responses", - "delete_run", - "delete_skill", - "disable_add_prefix_to_prompt", - "disable_add_transform_inline_image_block", - "disable_add_user_agent_to_request_tags", - "disable_aiohttp_transport", - "disable_aiohttp_trust_env", - "disable_anthropic_gemini_context_caching_transform", - "disable_cache", - "disable_copilot_system_to_assistant", - "disable_end_user_cost_tracking", - "disable_end_user_cost_tracking_prometheus_only", - "disable_hf_tokenizer_download", - "disable_stop_sequence_limit", - "disable_streaming_logging", - "disable_token_counter", - "disable_vertex_batch_output_transformation", - "docker_model_runner_models", - "dotenv", - "dotprompt", - "drop_params", - "dynamodb", - "dynamodb_table_name", - "elevenlabs_models", - "email", - "email_templates", - "embedding", - "empower_models", - "enable_anthropic_prompt_caching", - "enable_azure_ad_token_refresh", - "enable_cache", - "enable_caching_on_provider_specific_optional_params", - "enable_end_user_cost_tracking_prometheus_only", - "enable_gemini_default_thinking_level_low", - "enable_json_schema_validation", - "enable_key_alias_format_validation", - "enable_loadbalancing_on_batch_endpoints", - "enable_model_config_credential_overrides", - "enable_preview_features", - "enum", - "error_logs", - "evals", - "exception_type", - "exceptions", - "expose_router_debug_in_errors", - "extra_spend_tag_headers", - "failure_callback", - "fal_ai_models", - "fallbacks", - "featherless_ai_models", - "field_serializer", - "field_validator", - "file_content", - "file_content_streaming", - "file_delete", - "file_list", - "file_retrieve", - "files", - "filter_invalid_headers", - "filter_out_litellm_params", - "fine_tuning", - "fireworks_ai_embedding_models", - "fireworks_ai_models", - "flatten_form_field_values", - "flatten_unencrypted_web_search_results_in_anthropic_messages", - "force_ipv4", - "forward_traceparent_to_llm_provider", - "friendliai_models", - "function_call_prompt", - "futures", - "galadriel_models", - "galileo", - "gcs_bucket", - "gcs_pub_sub_use_v1", - "gcs_pubsub", - "gdc_api_base", - "gdc_key", - "gdc_transformation", - "gemini_live_defer_setup", - "gemini_models", - "generic_api", - "generic_api_use_v1", - "generic_logger_headers", - "get_agent", - "get_api_key_from_env", - "get_args", - "get_assistants", - "get_audio_file_for_health_check", - "get_azure_credentials", - "get_completion_messages", - "get_configured_request_timeout", - "get_content_from_model_response", - "get_eval", - "get_litellm_gateway_api_key", - "get_litellm_params", - "get_llm_provider", - "get_messages", - "get_messages_interceptors", - "get_mime_type", - "get_model_cost_map", - "get_model_info", - "get_non_default_completion_params", - "get_non_default_transcription_params", - "get_openai_credentials", - "get_optional_params", - "get_optional_params_add_message", - "get_optional_params_embeddings", - "get_optional_params_image_gen", - "get_optional_params_transcription", - "get_optional_rerank_params", - "get_requester_metadata", - "get_responses", - "get_run", - "get_secret", - "get_secret_bool", - "get_secret_str", - "get_skill", - "get_standard_openai_params", - "get_thread", - "get_type_hints", - "get_vertex_ai_model_route", - "gigachat_key", - "gigachat_models", - "github_copilot_models", - "global_bitbucket_config", - "global_disable_no_log_param", - "global_gitlab_config", - "google_batch_embeddings", - "google_genai", - "google_moderation_confidence_threshold", - "gradient_ai_api_key", - "gradient_ai_models", - "greenscale", - "groq_chat_completions", - "groq_key", - "groq_models", - "guardrail_name_config_map", - "headers", - "heapq", - "helicone", - "helicone_mock_client", - "heroku_key", - "heroku_models", - "heroku_transformation", - "httpx", - "huggingface_embed", - "huggingface_key", - "huggingface_models", - "humanloop", - "hyperbolic_models", - "identify", - "image_edit", - "image_generation", - "image_variation", - "images", - "importlib", - "in_memory_llm_clients_cache", - "inception_key", - "inception_models", - "include_cost_in_streaming_usage", - "infer_openai_data_residency", - "infinity_key", - "infinity_models", - "ingest", - "initialized_langfuse_clients", - "input_callback", - "inspect", - "integrations", - "interactions", - "internal_user_budget_duration", - "is_azure_document_intelligence_model", - "is_bedrock_pricing_only_model", - "is_openai_finetune_model", - "is_reasoning_auto_summary_enabled", - "jina_ai_models", - "json", - "json_logs", - "key_generation_settings", - "known_tokenizer_config", - "lago", - "lambda_ai_models", - "langfuse", - "langfuse_default_tags", - "langfuse_enable_update_trace_keys", - "langsmith", - "langsmith_batch_size", - "langsmith_mock_client", - "lemonade_key", - "lemonade_models", - "lemonade_transformation", - "list_agent_versions", - "list_agents", - "list_batches", - "list_container_files", - "list_containers", - "list_evals", - "list_fine_tuning_jobs", - "list_input_items", - "list_runs", - "list_skills", - "litellm", - "litellm_agent", - "litellm_completion_transformation_handler", - "litellm_core_utils", - "litellm_mode", - "literal_ai", - "llama_api_key", - "llama_models", - "llamagate_models", - "llamaguard_model_name", - "llamaguard_unsafe_content_categories", - "llm_guard_mode", - "llm_http_handler", - "llm_passthrough_route", - "llms", - "log_client_error_tracebacks", - "log_level", - "log_raw_request_response", - "logfire_logger", - "logged_real_time_event_types", - "logging", - "longer_context_model_fallback_dict", - "lunary", - "main", - "map_system_message_pt", - "maritalk_key", - "maritalk_models", - "max_budget", - "max_end_user_budget", - "max_end_user_budget_id", - "max_fallbacks", - "max_internal_user_budget", - "max_tokens", - "max_ui_session_budget", - "max_user_budget", - "maybe_run_chat_completion_agentic_loop", - "mcp_tool_search", - "mimetypes", - "minimax_models", - "mistral_chat_models", - "mlflow", - "mock_client_factory", - "mock_completion", - "mock_completion_streaming_obj", - "mock_embedding", - "mock_image_generation", - "mock_response", - "mock_responses_api_response", - "model_alias_map", - "model_cost", - "model_cost_map_url", - "model_fallbacks", - "model_group_settings", - "model_list", - "model_list_set", - "model_serializer", - "model_validator", - "models", - "models_by_provider", - "modelscope_models", - "moderation", - "modify_params", - "moonshot_models", - "morph_models", - "nebius_embedding_models", - "nebius_key", - "nebius_models", - "network_mock", - "newrelic", - "newrelic_params", - "nlp_cloud_chat_completion", - "nlp_cloud_key", - "nlp_cloud_models", - "novita_api_key", - "novita_models", - "nscale_models", - "num_retries", - "num_retries_per_request", - "nvidia_nim_models", - "nvidia_riva_audio_transcriptions", - "nvidia_riva_models", - "oci_models", - "oci_transformation", - "ocr", - "ollama", - "ollama_key", - "ollama_models", - "ollama_pt", - "oobabooga", - "open_ai_chat_completion_models", - "open_ai_embedding_models", - "open_ai_text_completion_models", - "openai", - "openai_assistants_api", - "openai_audio_transcriptions", - "openai_batches_instance", - "openai_chat_completions", - "openai_compatible_endpoints", - "openai_compatible_providers", - "openai_files_instance", - "openai_fine_tuning_apis_instance", - "openai_image_generation_models", - "openai_image_variations", - "openai_key", - "openai_like_chat_completion", - "openai_like_embedding", - "openai_like_key", - "openai_moderations_model_name", - "openai_text_completion_compatible_providers", - "openai_text_completions", - "openai_video_generation_models", - "openmeter", - "openrouter_key", - "openrouter_models", - "opentelemetry", - "opentelemetry_utils", - "opik", - "organization", - "os", - "otel", - "output_parse_pii", - "overload", - "override", - "overwrite_user_with_key_hash", - "ovhcloud_embedding_models", - "ovhcloud_key", - "ovhcloud_models", - "ovhcloud_transformation", - "palm", - "palm_models", - "parse_ocr_request_format", - "partial", - "passthrough", - "peek_reasoning_summary_aliases", - "perplexity_models", - "petals_handler", - "petals_models", - "post_call_rules", - "posthog", - "posthog_mock_client", - "pre_call_rules", - "pre_process_non_default_params", - "predibase_chat_completions", - "predibase_key", - "predibase_tenant_id", - "presidio_ad_hoc_recognizers", - "print_verbose", - "priority_reservation", - "project", - "prometheus_deployment_and_latency_caller_identity", - "prometheus_emit_rate_limit_labels", - "prometheus_emit_stream_label", - "prometheus_end_user_metrics_cleanup_interval_seconds", - "prometheus_end_user_metrics_max_series_per_metric", - "prometheus_end_user_metrics_ttl_seconds", - "prometheus_exclude_labels", - "prometheus_exclude_metrics", - "prometheus_initialize_budget_metrics", - "prometheus_latency_buckets", - "prometheus_metrics_config", - "prometheus_user_budget_label_include_email_alias", - "prompt_factory", - "prompt_layer", - "prompt_management_base", - "prompt_name_config_map", - "provider_url_destination_allowed_hosts", - "proxy", - "proxy_auth", - "public_agent_groups", - "public_mcp_hub_strict_whitelist", - "public_mcp_servers", - "public_model_groups", - "public_model_groups_links", - "publicai_models", - "query", - "qwen_ai_platform_models", - "qwencloud_models", - "rag", - "random", - "re", - "read_config_args", - "realtime_api", - "reasoning_auto_summary", - "recraft_models", - "redact_messages_in_exceptions", - "redact_user_api_key_info", - "reducto_models", - "replicate_chat_completion", - "replicate_key", - "replicate_models", - "repositories", - "request_correlation_in_logs", - "request_timeout", - "request_timeout_explicitly_set", - "require_auth_for_metrics_endpoint", - "require_managed_files", - "rerank", - "rerank_api", - "responses", - "responses_api_bridge_check", - "responses_with_retries", - "retrieve_batch", - "retrieve_container", - "retrieve_fine_tuning_job", - "retry", - "return_response_headers", - "route_all_chat_openai_to_responses", - "router", - "router_strategy", - "router_utils", - "run_async_function", - "run_server", - "run_thread", - "run_thread_stream", - "runtime_checkable", - "runwayml_models", - "rust", - "rust_bridge", - "rust_ocr_bridge", - "s3", - "s3_audit_callback_params", - "s3_callback_params", - "s3_v2", - "safe_deep_copy", - "safe_memory_mode", - "sagemaker_chat_completion", - "sagemaker_llm", - "sambanova_embedding_models", - "sambanova_models", - "sandbox", - "sanitize_tool_use_ids_in_anthropic_messages", - "sap_gen_ai_hub_chat_completions", - "sap_gen_ai_hub_emb", - "sap_service_key", - "scheduler", - "search", - "secret_manager_client", - "secret_managers", - "service_callback", - "set_global_bitbucket_config", - "set_global_gitlab_config", - "set_verbose", - "should_run_mock_completion", - "skills", - "skip_system_message_in_guardrail", - "skip_tool_message_in_guardrail", - "snowflake_key", - "snowflake_models", - "soniox_models", - "speech", - "sqs", - "sse_keepalive_ping_interval_seconds", - "ssl_certificate", - "ssl_ecdh_curve", - "ssl_security_level", - "ssl_verify", - "stability_models", - "standard_logging_payload_excluded_fields", - "store_audit_logs", - "stream_chunk_builder", - "stream_chunk_builder_text_completion", - "stringify_json_tool_call_content", - "strip_anthropic_total_tokens", - "strip_empty_content_blocks_from_anthropic_messages", - "strip_reasoning_summary_aliases_from_optional_params", - "success_callback", - "supabase", - "supports_httpx_timeout", - "suppress_debug_info", - "sys", - "tag_budget_config", - "telemetry", - "tencent_models", - "text_completion", - "text_completion_codestral_models", - "text_completion_inception_models", - "threading", - "tiktoken", - "time", - "together_ai_models", - "together_rerank", - "togetherai_api_key", - "token", - "token_counter", - "traceback", - "traceloop", - "tracer", - "transcription", - "turn_off_message_logging", - "types", - "updateDeployment", - "updateLiteLLMParams", - "update_cache", - "update_messages_with_model_file_ids", - "update_responses_input_with_model_file_ids", - "update_responses_tools_with_model_file_ids", - "upload_container_file", - "upperbound_key_generate_params", - "urlsplit", - "use_aiohttp_transport", - "use_chat_completions_url_for_anthropic_messages", - "use_client", - "use_legacy_interactions_schema", - "use_litellm_proxy", - "user_url_allowed_hosts", - "user_url_validation", - "utils", - "uuid", - "uuid_module", - "v0_models", - "validate_and_fix_openai_messages", - "validate_and_fix_openai_tools", - "validate_and_fix_thinking_param", - "validate_anthropic_api_metadata", - "validate_chat_completion_tool_choice", - "validate_end_user_id_in_db", - "validate_openai_optional_params", - "vector_store_file_content", - "vector_store_file_create", - "vector_store_file_delete", - "vector_store_file_list", - "vector_store_file_retrieve", - "vector_store_file_update", - "vector_store_files", - "vector_store_index_registry", - "vector_store_registry", - "vector_stores", - "verbose_logger", - "vercel_ai_gateway_key", - "vercel_ai_gateway_models", - "vertexAITextEmbeddingConfig", - "vertex_ai_ai21_models", - "vertex_ai_batches_instance", - "vertex_ai_files_instance", - "vertex_ai_image_models", - "vertex_ai_non_gemini", - "vertex_ai_safety_settings", - "vertex_ai_video_models", - "vertex_anthropic_models", - "vertex_chat_completion", - "vertex_chat_models", - "vertex_code_chat_models", - "vertex_code_text_models", - "vertex_deepseek_models", - "vertex_embedding", - "vertex_embedding_models", - "vertex_fine_tuning_apis_instance", - "vertex_gemma_chat_completion", - "vertex_image_generation", - "vertex_language_models", - "vertex_llama3_models", - "vertex_location", - "vertex_minimax_models", - "vertex_mistral_models", - "vertex_model_garden_chat_completion", - "vertex_moonshot_models", - "vertex_multimodal_embedding", - "vertex_openai_models", - "vertex_partner_models_chat_completion", - "vertex_project", - "vertex_text_models", - "vertex_vision_models", - "vertex_zai_models", - "video_content", - "video_create_character", - "video_edit", - "video_extension", - "video_generation", - "video_get_character", - "video_list", - "video_remix", - "video_status", - "videos", - "vllm_handler", - "volcengine_models", - "voyage_models", - "wait", - "wandb_key", - "wandb_models", - "warnings", - "watsonx_chat_completion", - "watsonx_models", - "xai_key", - "xai_models", - "zai_models", -) diff --git a/litellm/proxy/__init__.py b/litellm/proxy/__init__.py index dc819fbc85c..b6e690fd591 100644 --- a/litellm/proxy/__init__.py +++ b/litellm/proxy/__init__.py @@ -1,11 +1 @@ -from types import ModuleType -from typing import Final - - -def __getattr__(name: str) -> ModuleType: - from litellm._lazy_imports import lazy_import_submodule - - submodule: Final = lazy_import_submodule(__name__, name) - if submodule is not None: - return submodule - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +from . import * diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 7254b05db2e..d6276c03155 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -254,7 +254,7 @@ class _ProxyDBLogger(CustomLogger): key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None)) end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None) sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None) - response_cost = ( + response_cost: Final = ( sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None) ) tags: Final = _get_request_tags_for_cost_tracking( @@ -269,10 +269,6 @@ class _ProxyDBLogger(CustomLogger): if response_cost is not None: user_api_key: Final = metadata.get("user_api_key", None) - if kwargs.get("cache_hit", False) is True: - response_cost = 0.0 - verbose_proxy_logger.debug("Cache Hit: response_cost %s, for user_id %s", response_cost, user_id) - verbose_proxy_logger.debug( "user_api_key %s, user_id %s, team_id %s, end_user_id %s", user_api_key, diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 14b2b4e3299..89c04208d65 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -77,7 +77,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover The seam is `provider_edge.py`: `start_provider_edge` boots an in-process HTTP server (one shared instance per pytest process, `e2e_config.provider_edge_base` is the accessor) that mounts each supported provider under a path prefix (`EDGE_MOUNTS`: `/openai` -> `https://api.openai.com`, `/anthropic` -> `https://api.anthropic.com`). A test participates by registering its deployment with `api_base=provider_edge_base("openai")` plus the provider's path suffix; `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` is the reference. In live mode the accessor returns None and the deployment defaults to the real provider, so an edge-wired test runs in all three modes unchanged. Non-wired tests hit their providers live in every mode. The edge binds `E2E_PROVIDER_EDGE_BIND_HOST` (default 127.0.0.1) and advertises `E2E_PROVIDER_EDGE_ADVERTISE_HOST` in the api_base it hands out, for proxies running in containers -A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. Responses come in two shapes told apart by a `kind` tag: an ordinary one holding a single base64 body, and, for a response the provider streamed (`content-type: text/event-stream`), one holding its transfer chunks in order plus why the stream ended early if it did, so replay reproduces the split points the provider chose instead of one coalesced body. `fixture_bundle.py` owns the format, and `BUNDLE_FORMAT_VERSION` is checked on load, so a bundle recorded under older rules is refused by name rather than partially read. Record serves the proxy the same filtered stored response replay will serve later, chunk for chunk on a stream, so the two modes are byte-identical from the proxy's side of the socket +A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. Responses come in two shapes told apart by a `kind` tag: an ordinary one holding a single base64 body, and, for a response the provider streamed (`content-type: text/event-stream`), one holding its transfer chunks in order plus why the stream ended early if it did, so replay reproduces the split points the provider chose instead of one coalesced body. `fixture_bundle.py` owns the format, and `BUNDLE_FORMAT_VERSION` is checked on load, so a bundle recorded under older rules is refused by name rather than partially read. Record serves the proxy the same filtered stored response replay will serve later, chunk for chunk on a stream, so the two modes are byte-identical from the proxy's side of the socket. Replay does not reproduce the provider's inter-chunk timing (chunks go out as fast as the socket takes them), so a test that judges streaming on the clock, such as the `stream_event_arrivals` lead between the first content delta and `message_stop`, gates that assertion on `provider_paces_stream()` and proves only the event grammar in replay Multipart identity is the fiddly corner, and the rules exist because each one had a collision behind it. A part counts as an upload when it carries a filename or declares its own content type, and everything else is an ordinary field. Field names get a `name[n]` suffix on repeats, with a literal `[` doubled first, so a form that repeats `purpose` never keys the same as one that literally sends `purpose[1]`. A field whose name reads as a credential is stored as ``, which stays key-preserving because the key is recomputed from the stored request rather than saved alongside it, so the live request carrying the real value still matches its redacted fixture. A field value that is not UTF-8 is stored as a base64 sha256 digest, base64 and not hex because the canonicalizer rewrites any 64-character hex run to `` and would fold every binary value onto one key. The uploaded parts contribute a JSON list rather than a `field:filename` string, so a separator inside a filename cannot impersonate a field boundary, and their byte length is stored for a reader's benefit but deliberately left out of the key, since the canonicalizer absorbs timestamp and id drift inside a file that changes its length diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 21c5a338dc3..09d17b9a0db 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -10,6 +10,7 @@ import os import time import uuid from pathlib import Path +from typing import Final from dotenv import load_dotenv @@ -192,6 +193,13 @@ def provider_edge_base(mount: str) -> str | None: ) +STREAM_MIN_LEAD_SECONDS: Final = 1.0 + + +def provider_paces_stream() -> bool: + return parse_fixture_mode(FIXTURE_MODE_RAW) != "replay" + + def unique_marker() -> str: """A short unique token per call/run, so concurrent runs and the shared response cache never collide on prompts, tags, or customer ids. In record diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index bc76eb3ea7a..9d5f1658e91 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -16,9 +16,9 @@ requests itself imports. from __future__ import annotations import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import Generator, Generic, Iterator, Literal, NewType, Protocol, TypeVar, cast +from typing import Final, Generator, Generic, Iterator, Literal, NewType, Protocol, TypeVar, cast import pytest import requests @@ -142,6 +142,7 @@ class StreamingResponse(BaseModel): body: str chunks: int = 0 # streamed events (0 for non-streaming) stream_events: list[str] = [] + stream_event_arrivals: list[float] = [] # First in-stream error event, if any. A streamed call commits its HTTP 200 # before the upstream completes, so upstream failures (e.g. insufficient # quota) arrive as SSE error events inside an otherwise-successful response; @@ -184,7 +185,20 @@ class BinaryStream(BaseModel): return "chunked" in (self.transfer_encoding or "") -def _hdr(resp: requests.Response, name: str) -> str | None: +class SseResponse(Protocol): + @property + def status_code(self) -> int: ... + + @property + def headers(self) -> Mapping[str, str]: ... + + @property + def text(self) -> str: ... + + def iter_lines(self) -> Iterator[bytes]: ... + + +def _hdr(resp: SseResponse, name: str) -> str | None: value = resp.headers.get(name) return value if isinstance(value, str) else None @@ -457,7 +471,7 @@ def probe( return ProbeResult(status_code=resp.status_code, body=resp.text) -def _parse_response_cost(resp: requests.Response) -> float | None: +def _parse_response_cost(resp: SseResponse) -> float | None: raw = _hdr(resp, "x-litellm-response-cost") if raw is None or raw == "": return None @@ -467,11 +481,26 @@ def _parse_response_cost(resp: requests.Response) -> float | None: return None -def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingResponse: - call_id = _hdr(resp, "x-litellm-call-id") - response_cost = _parse_response_cost(resp) - content_type = _hdr(resp, "content-type") - headers = {name.lower(): value for name, value in resp.headers.items()} +_SSE_DATA_PREFIX: Final = b"data: " +_SSE_DONE: Final = "[DONE]" + + +def _is_stream_error_line(line: bytes) -> bool: + return ( + line.startswith(b"event: error") + or b'"type":"error"' in line + or b'"type": "error"' in line + or line.startswith(b'data: {"error"') + ) + + +def streaming_outcome( + resp: SseResponse, stream: bool, *, sent_at: float, clock: Callable[[], float] = time.monotonic +) -> StreamingResponse: + call_id: Final = _hdr(resp, "x-litellm-call-id") + response_cost: Final = _parse_response_cost(resp) + content_type: Final = _hdr(resp, "content-type") + headers: Final = {name.lower(): value for name, value in resp.headers.items()} if not stream or not (200 <= resp.status_code < 300): return StreamingResponse( status_code=resp.status_code, @@ -481,29 +510,13 @@ def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingRespon headers=headers, body=resp.text, ) - lines = cast("Iterator[bytes]", resp.iter_lines()) - chunks = 0 - stream_error: str | None = None - stream_events: list[str] = [] - stream_done = False - for line in lines: - if not line: - continue - chunks += 1 - decoded_line = line.decode(errors="replace") - if decoded_line.startswith("data: "): - payload = decoded_line.removeprefix("data: ") - if payload == "[DONE]": - stream_done = True - else: - stream_events.append(payload) - if stream_error is None and ( - line.startswith(b"event: error") - or b'"type":"error"' in line - or b'"type": "error"' in line - or line.startswith(b'data: {"error"') - ): - stream_error = line.decode(errors="replace")[:300] + stamped: Final = tuple((line, clock() - sent_at) for line in resp.iter_lines() if line) + payloads: Final = tuple( + (line.removeprefix(_SSE_DATA_PREFIX).decode(errors="replace"), arrived) + for line, arrived in stamped + if line.startswith(_SSE_DATA_PREFIX) + ) + events: Final = tuple((payload, arrived) for payload, arrived in payloads if payload != _SSE_DONE) return StreamingResponse( status_code=resp.status_code, call_id=call_id, @@ -511,10 +524,14 @@ def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingRespon content_type=content_type, headers=headers, body="", - chunks=chunks, - stream_events=stream_events, - stream_done=stream_done, - stream_error=stream_error, + chunks=len(stamped), + stream_events=[payload for payload, _ in events], + stream_event_arrivals=[arrived for _, arrived in events], + stream_done=any(payload == _SSE_DONE for payload, _ in payloads), + stream_error=next( + (line.decode(errors="replace")[:300] for line, _ in stamped if _is_stream_error_line(line)), + None, + ), ) @@ -531,6 +548,7 @@ def send( x-litellm-call-id header. For native/passthrough bodies and for calls judged by status rather than a typed JSON model (e.g. a budget block is a non-2xx). With ``stream=True`` the SSE body is consumed and its events counted instead.""" + sent_at: Final = time.monotonic() try: resp = request_with_retry( lambda: requests.post( @@ -544,7 +562,7 @@ def send( ) except requests.RequestException as exc: return StreamingResponse(status_code=-1, body=str(exc)) - return _streaming_outcome(resp, stream) + return streaming_outcome(resp, stream, sent_at=sent_at) def stream( diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index a5f36a8cbdd..c731b52acd8 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -8,8 +8,15 @@ litellm-regression-tests/tests/test_inference_endpoints.py. from __future__ import annotations +from typing import Final + import pytest -from e2e_config import provider_edge_base, unique_marker +from e2e_config import ( + STREAM_MIN_LEAD_SECONDS, + provider_edge_base, + provider_paces_stream, + unique_marker, +) from e2e_http import assert_client_error, require_successful_call, unwrap from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager @@ -159,19 +166,21 @@ class TestAnthropicMessages: """Edge-wired like its non-streaming siblings, so record and replay both carry the streamed response. - Asserts the shape of the event sequence, not just that deltas and a stop - appeared somewhere in it: the answer arrives across several deltas, and the - usage event sits between the last of them and ``message_stop``. A replay that - coalesced the response into one buffered body could not satisfy either.""" + Asserts what the proxy controls: the event grammar (usage between the last + content delta and ``message_stop``) and, on the clock, that the relay is + incremental. How many deltas a reply is split into is the provider's choice, so + the first content delta must instead reach the client well before + ``message_stop``, which a buffered response cannot do. Replay serves chunks back + to back, so only live and record runs judge the timing.""" model, key = self._register(endpoints_client, resources) result = endpoints_client.proxy.messages_stream( key, AnthropicMessagesBody( model=model, - max_tokens=400, + max_tokens=800, stream=True, - messages=[ChatMessage(role="user", content="Count from 1 to 100, one number per line.")], + messages=[ChatMessage(role="user", content="Count from 1 to 200, one number per line.")], ), ) require_successful_call(result) @@ -186,10 +195,7 @@ class TestAnthropicMessages: delta_positions = [ index for index, event in enumerate(events) if event.type == "content_block_delta" ] - assert len(delta_positions) >= 2, ( - f"stream carried {len(delta_positions)} content deltas, so it was not " - f"incremental: {types}" - ) + assert delta_positions, f"stream carried no content deltas: {types}" text = "".join( event.delta.text for event in events @@ -209,6 +215,15 @@ class TestAnthropicMessages: f"usage did not land between the last content delta and message_stop: {types}" ) + first_delta_at: Final = result.stream_event_arrivals[delta_positions[0]] + stop_at: Final = result.stream_event_arrivals[stop_position] + if provider_paces_stream(): + assert stop_at - first_delta_at >= STREAM_MIN_LEAD_SECONDS, ( + f"first content delta reached the client {first_delta_at:.2f}s after the request " + f"and message_stop {stop_at:.2f}s after it; a relayed stream shows the first delta " + f"at least {STREAM_MIN_LEAD_SECONDS}s before the end, so the response was buffered" + ) + @pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works") def test_messages_tool_use( self, endpoints_client: EndpointsClient, resources: ResourceManager diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py index 2c8a7a3aa20..31e74c22e17 100644 --- a/tests/e2e/llm_translation/test_together_ai_e2e.py +++ b/tests/e2e/llm_translation/test_together_ai_e2e.py @@ -23,7 +23,7 @@ from datetime import date from typing import Final import pytest -from e2e_config import unique_marker +from e2e_config import STREAM_MIN_LEAD_SECONDS, provider_paces_stream, unique_marker from e2e_http import StreamingResponse, require_successful_call, unwrap from lifecycle import ResourceManager from models import ( @@ -81,7 +81,7 @@ PERSON_RESPONSE_FORMAT: dict[str, object] = { } WEATHER_PROMPT = "What is the weather in Paris? Use the tool." WEATHER_REPORT = "Paris: 22 degrees Celsius, clear skies, wind from the northwest at 9 km/h" -COUNTING_PROMPT = "Count from 1 to 20, one number per line." +COUNTING_PROMPT = "Count from 1 to 200, one number per line." WEATHER_TOOL = ChatTool( function=ChatToolFunction( @@ -753,7 +753,7 @@ class TestTogetherMessages: key, AnthropicMessagesBody( model=model, - max_tokens=512, + max_tokens=2048, stream=True, messages=[ChatMessage(role="user", content=COUNTING_PROMPT)], ), @@ -763,11 +763,24 @@ class TestTogetherMessages: assert not result.stream_error, f"stream errored: {result.stream_error}" events = [_MessagesStreamEvent.model_validate_json(event) for event in result.stream_events] types = [event.type for event in events] - text_deltas = [ + delta_positions = [ + index for index, event in enumerate(events) if event.type == "content_block_delta" + ] + assert delta_positions, f"stream carried no content deltas: {types}" + text = "".join( event.delta.text for event in events - if event.type == "content_block_delta" and event.delta is not None and event.delta.text - ] - assert len(text_deltas) >= 2, f"stream was not incremental: {types}" - assert "20" in "".join(text_deltas), f"streamed text lost the answer: {text_deltas}" + if event.type == "content_block_delta" and event.delta is not None + ) + assert "200" in text, f"streamed text lost the answer: {text[:300]!r}" assert "message_stop" in types, f"stream never reached message_stop: {types}" + + stop_position: Final = types.index("message_stop") + first_delta_at: Final = result.stream_event_arrivals[delta_positions[0]] + stop_at: Final = result.stream_event_arrivals[stop_position] + if provider_paces_stream(): + assert stop_at - first_delta_at >= STREAM_MIN_LEAD_SECONDS, ( + f"first content delta reached the client {first_delta_at:.2f}s after the request " + f"and message_stop {stop_at:.2f}s after it; a relayed stream shows the first delta " + f"at least {STREAM_MIN_LEAD_SECONDS}s before the end, so the response was buffered" + ) diff --git a/tests/e2e/test_e2e_http.py b/tests/e2e/test_e2e_http.py index 007801a797a..66841725d1d 100644 --- a/tests/e2e/test_e2e_http.py +++ b/tests/e2e/test_e2e_http.py @@ -12,12 +12,14 @@ monkeypatches anything. from __future__ import annotations -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final import pytest -from e2e_http import RETRY_ATTEMPTS, TRANSIENT_STATUSES, request_with_retry +from e2e_http import RETRY_ATTEMPTS, TRANSIENT_STATUSES, request_with_retry, streaming_outcome @dataclass @@ -80,3 +82,55 @@ class TestTransientRetryPolicy: assert result is responses[RETRY_ATTEMPTS - 1] assert sleep.delays == [0.5, 1.0] assert [r.close_calls for r in responses] == [1, 1, 0, 0] + + +@dataclass(frozen=True, slots=True) +class FakeSseResponse: + lines: Sequence[bytes] + status_code: int = 200 + headers: Mapping[str, str] = MappingProxyType({"content-type": "text/event-stream"}) + text: str = "" + + def iter_lines(self) -> Iterator[bytes]: + return iter(self.lines) + + +def _ticking_clock(start: float, step: float) -> Callable[[], float]: + ticks: Final = iter(range(10_000)) + return lambda: start + step * next(ticks) + + +class TestStreamEventArrivals: + def test_each_event_is_stamped_at_the_moment_its_line_arrives(self) -> None: + resp: Final = FakeSseResponse( + lines=( + b"event: message_start", + b'data: {"type":"message_start"}', + b"", + b"event: ping", + b'data: {"type":"ping"}', + b"event: content_block_delta", + b'data: {"type":"content_block_delta"}', + b"data: [DONE]", + ) + ) + + result: Final = streaming_outcome(resp, True, sent_at=100.0, clock=_ticking_clock(start=100.0, step=0.5)) + + assert result.stream_events == [ + '{"type":"message_start"}', + '{"type":"ping"}', + '{"type":"content_block_delta"}', + ] + assert result.stream_event_arrivals == [0.5, 1.5, 2.5] + assert result.stream_done + assert result.chunks == 7 + + def test_a_non_streaming_outcome_carries_no_arrivals(self) -> None: + resp: Final = FakeSseResponse(lines=(), status_code=400, text="bad request") + + result: Final = streaming_outcome(resp, True, sent_at=0.0, clock=_ticking_clock(start=0.0, step=1.0)) + + assert result.stream_events == [] + assert result.stream_event_arrivals == [] + assert result.body == "bad request" diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 8043a1aca3f..f3fd9f41cc2 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -1783,6 +1783,53 @@ async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata(): ) +@pytest.mark.asyncio +async def test_track_cost_callback_keeps_guardrail_cost_on_cache_hit(): + """A cache hit skips the LLM, not the guardrail that screened the prompt, so the + guardrail's provider charge must still reach spend logs and budgets. The payload + already prices the LLM share at 0 on a cache hit, so its response_cost is the + guardrail cost alone and the callback must pass it through untouched.""" + logger = _ProxyDBLogger() + kwargs = { + "call_type": "acompletion", + "model": "gpt-4o", + "cache_hit": True, + "response_cost": 0.0, + "litellm_params": { + "metadata": { + "user_api_key": "hashed-key", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "team-1", + } + }, + "standard_logging_object": { + "response_cost": 0.0003, + "request_tags": [], + "metadata": {}, + "cost_breakdown": {"guardrail_cost": 0.0003, "total_cost": 0.0003}, + }, + } + + with ( + patch("litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock) as mock_increment, # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam + patch("litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock), # test-quality-ok: same function-body import, no injection seam + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging, # test-quality-ok: same function-body import, no injection seam + ): + mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock() + mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock() + + await logger._PROXY_track_cost_callback( + kwargs=kwargs, + completion_response={"id": "cached-call-1"}, + start_time=datetime.now(), + end_time=datetime.now(), + ) + + update_kwargs = mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs + assert update_kwargs["response_cost"] == pytest.approx(0.0003) + assert mock_increment.call_args.kwargs["response_cost"] == pytest.approx(0.0003) + + @pytest.mark.parametrize( "call_type, expected", [ diff --git a/tests/test_litellm/test_lazy_imports.py b/tests/test_litellm/test_lazy_imports.py index 09d5e23e2e6..2b16a812611 100644 --- a/tests/test_litellm/test_lazy_imports.py +++ b/tests/test_litellm/test_lazy_imports.py @@ -1,8 +1,5 @@ """Simple tests for lazy import functionality.""" -import importlib -import json -import subprocess import sys import pytest @@ -10,10 +7,6 @@ import pytest import litellm from litellm._lazy_imports import ( - _SDK_MODULE_ALIASES, - _SDK_SYMBOLS_IMPORT_MAP, - lazy_import_litellm_submodule, - _lazy_import_sdk_symbols, COST_CALCULATOR_NAMES, LITELLM_LOGGING_NAMES, UTILS_NAMES, @@ -353,83 +346,3 @@ def test_utils_module_lazy_imports(): assert name in utils_globals _verify_only_requested_name_imported_in_utils(name, UTILS_MODULE_NAMES) - - -def test_sdk_symbols_lazy_imports(): - """Every symbol previously imported eagerly in litellm/__init__.py resolves to the source module attribute.""" - for name, (module_path, attr_name) in _SDK_SYMBOLS_IMPORT_MAP.items(): - resolved = getattr(litellm, name) - expected = getattr(importlib.import_module(module_path), attr_name) - assert resolved is expected, f"litellm.{name} is not {module_path}.{attr_name}" - - -def test_sdk_module_aliases(): - """Module-valued attributes (litellm.anthropic, litellm.httpx, ...) resolve to the aliased modules.""" - for name, module_path in _SDK_MODULE_ALIASES.items(): - assert getattr(litellm, name) is importlib.import_module(module_path) - - -def test_litellm_submodule_fallback(): - """litellm. attribute access resolves real submodules and returns None for unknown names.""" - assert lazy_import_litellm_submodule("budget_manager") is importlib.import_module("litellm.budget_manager") - assert litellm.utils is importlib.import_module("litellm.utils") - assert lazy_import_litellm_submodule("not_a_real_submodule") is None - with pytest.raises(AttributeError): - _ = litellm.not_a_real_attribute - - -def test_missing_attribute_stays_attribute_error_when_find_spec_lies(monkeypatch): - """getattr(litellm, name, default) must not leak ModuleNotFoundError when find_spec is patched to always succeed.""" - monkeypatch.setattr(importlib.util, "find_spec", lambda name: object()) - assert getattr(litellm, "not_a_real_submodule", None) is None - with pytest.raises(AttributeError): - _ = litellm.not_a_real_attribute - - -def test_proxy_private_submodule_resolves_in_fresh_process(): - """litellm.proxy._types resolves without an eager proxy import (used by documentation checks).""" - code = "import litellm\nprint(litellm.proxy._types.__name__)\n" - result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) - assert result.returncode == 0, result.stderr - assert result.stdout.strip() == "litellm.proxy._types" - - -def test_lazy_instances_are_singletons(): - """Lazily created instances are cached, so repeated access returns the same object.""" - assert litellm._key_management_settings is litellm._key_management_settings - assert litellm.vertexAITextEmbeddingConfig is litellm.vertexAITextEmbeddingConfig - from litellm.types.secret_managers.main import KeyManagementSettings - - assert isinstance(litellm._key_management_settings, KeyManagementSettings) - - -def test_star_import_exports_public_api(): - """`from litellm import *` keeps exporting the full public surface despite lazy loading.""" - code = ( - "from litellm import *\n" - "import litellm\n" - "missing = [n for n in litellm.__all__ if n not in dir()]\n" - "assert not missing, missing[:20]\n" - "assert callable(completion) and callable(Router)\n" - ) - result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) - assert result.returncode == 0, result.stderr - - -@pytest.mark.skipif(sys.platform != "linux", reason="reads /proc for RSS") -def test_import_litellm_stays_lightweight(): - """`import litellm` must not pull in the SDK/proxy heavyweights or blow up RSS (LIT-6607).""" - code = ( - "import json, re, sys\n" - "import litellm\n" - "heavy = [m for m in ('litellm.main', 'litellm.utils', 'litellm.router', 'litellm.proxy.proxy_cli',\n" - " 'tiktoken', 'fastapi', 'grpc', 'boto3') if m in sys.modules]\n" - "with open('/proc/self/status') as f:\n" - " rss_kb = int(re.search(r'VmRSS:\\s+(\\d+) kB', f.read()).group(1))\n" - "print(json.dumps({'total': len(sys.modules), 'heavy': heavy, 'rss_mb': rss_kb / 1024}))\n" - ) - result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True) - stats = json.loads(result.stdout) - assert stats["heavy"] == [], f"heavy modules imported eagerly: {stats['heavy']}" - assert stats["total"] < 800, f"import litellm loaded {stats['total']} modules" - assert stats["rss_mb"] < 75, f"import litellm used {stats['rss_mb']:.1f} MB RSS" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index 26d595f4d74..7702488f5bf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -424,17 +424,14 @@ describe("UsagePage", () => { // Check that key metrics are displayed const totalRequestElements = screen.getAllByText("Total Requests"); expect(totalRequestElements.length).toBeGreaterThan(0); - expect(screen.getByText("1,500")).toBeInTheDocument(); const successfulRequestLabelElements = screen.getAllByText("Successful Requests"); expect(successfulRequestLabelElements.length).toBeGreaterThan(0); - // Successful and Failed Requests both read the gateway counter, not the - // spend-derived 1,450 / 50 that the same payload carries for the per-key and - // per-model breakdowns. They must share a source, or the tiles contradict the - // endpoint breakdown chart below them. await waitFor(() => { expect(screen.getAllByText("424,242").length).toBeGreaterThan(0); }); expect(screen.getAllByText("909").length).toBeGreaterThan(0); + expect(screen.getByText("425,151")).toBeInTheDocument(); + expect(screen.queryByText("1,500")).not.toBeInTheDocument(); expect(screen.queryByText("1,450")).not.toBeInTheDocument(); }); @@ -454,7 +451,7 @@ describe("UsagePage", () => { renderWithProviders(); await waitFor(() => { - expect(screen.getAllByText("1,500").length).toBeGreaterThan(0); + expect(screen.getAllByText("75,000").length).toBeGreaterThan(0); }); await act(async () => { @@ -464,13 +461,13 @@ describe("UsagePage", () => { await waitFor(() => { expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledTimes(2); }); - expect(screen.queryByText("1,500")).not.toBeInTheDocument(); + expect(screen.queryByText("75,000")).not.toBeInTheDocument(); await act(async () => { releaseSecondFetch(); }); await waitFor(() => { - expect(screen.getAllByText("1,500").length).toBeGreaterThan(0); + expect(screen.getAllByText("75,000").length).toBeGreaterThan(0); }); }); @@ -485,8 +482,10 @@ describe("UsagePage", () => { await waitFor(() => { expect(screen.getAllByText("1,450").length).toBeGreaterThan(0); }); + expect(screen.getByText("1,500")).toBeInTheDocument(); expect(screen.queryByText("424,242")).not.toBeInTheDocument(); expect(screen.queryByText("909")).not.toBeInTheDocument(); + expect(screen.queryByText("425,151")).not.toBeInTheDocument(); expect(screen.queryByTestId("gateway-requests-by-endpoint")).not.toBeInTheDocument(); }); @@ -499,6 +498,7 @@ describe("UsagePage", () => { expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalled(); }); expect(mockGatewayDailyActivityCall).not.toHaveBeenCalled(); + expect(screen.getByText("1,500")).toBeInTheDocument(); expect(screen.queryByText("424,242")).not.toBeInTheDocument(); expect(screen.queryByTestId("gateway-requests-by-endpoint")).not.toBeInTheDocument(); }); @@ -1045,7 +1045,7 @@ describe("UsagePage", () => { }); // Should still render the data from the paginated fallback, which lands a render after the call - expect(await screen.findByText("1,500")).toBeInTheDocument(); + expect(await screen.findByText("75,000")).toBeInTheDocument(); }); it("should stop showing the previous range's paginated pages while a new range is in flight", async () => { @@ -1069,7 +1069,7 @@ describe("UsagePage", () => { renderWithProviders(); await waitFor(() => { - expect(screen.getAllByText("1,500").length).toBeGreaterThan(0); + expect(screen.getAllByText("75,000").length).toBeGreaterThan(0); }); await act(async () => { @@ -1079,13 +1079,13 @@ describe("UsagePage", () => { await waitFor(() => { expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledTimes(2); }); - expect(screen.queryByText("1,500")).not.toBeInTheDocument(); + expect(screen.queryByText("75,000")).not.toBeInTheDocument(); await act(async () => { releaseSecondAggregated(); }); await waitFor(() => { - expect(screen.getAllByText("1,500").length).toBeGreaterThan(0); + expect(screen.getAllByText("75,000").length).toBeGreaterThan(0); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index 29a81e1ae3f..a92d1209567 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -573,7 +573,10 @@ const UsagePage: React.FC = ({ teams, organizations }) => {

Total Requests

- {userSpendData.metadata?.total_api_requests?.toLocaleString() || 0} + {(gatewayActivity + ? gatewayActivity.total_successful_requests + gatewayActivity.total_failed_requests + : userSpendData.metadata?.total_api_requests + )?.toLocaleString() || 0}

diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 7f349fa29ba..634adc6fba8 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -32,7 +32,6 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Copy, Inbox, Search as SearchIcon, X } from "lucide-react"; -import { useRouter } from "next/navigation"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { prism } from "react-syntax-highlighter/dist/esm/styles/prism"; @@ -72,7 +71,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, const [modelHubData, setModelHubData] = useState(null); const [loading, setLoading] = useState(true); const [isModalVisible, setIsModalVisible] = useState(false); - const [isPublicPageModalVisible, setIsPublicPageModalVisible] = useState(false); const [selectedModel, setSelectedModel] = useState(null); const [filteredData, setFilteredData] = useState([]); const [isMakePublicModalVisible, setIsMakePublicModalVisible] = useState(false); @@ -93,7 +91,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, const [skillHubData, setSkillHubData] = useState([]); const [skillLoading, setSkillLoading] = useState(false); const [isMakeSkillPublicModalVisible, setIsMakeSkillPublicModalVisible] = useState(false); - const router = useRouter(); const { data: uiSettings, isLoading: isUISettingsLoading } = useUISettings(); // Check authentication requirement for public AI Hub @@ -256,10 +253,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, setIsMcpModalVisible(true); }, []); - const goToPublicModelPage = () => { - router.replace(`/model_hub_table?key=${accessToken}`); - }; - const handleMakePublicPage = () => { if (!accessToken) { return; @@ -289,7 +282,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, const handleOk = () => { setIsModalVisible(false); - setIsPublicPageModalVisible(false); setSelectedModel(null); setIsAgentModalVisible(false); setSelectedAgent(null); @@ -299,7 +291,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, const handleCancel = () => { setIsModalVisible(false); - setIsPublicPageModalVisible(false); setSelectedModel(null); setIsAgentModalVisible(false); setSelectedAgent(null); @@ -639,26 +630,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, )} - {/* Public Page Modal */} - !open && handleCancel()}> - - - {"Public Model Hub"} - -
-
-

Shareable Link:

-

- {`${getProxyBaseUrl()}/ui/model_hub_table`} -

-
-
- -
-
- - - {/* Model Details Modal */} !open && handleCancel()}>