From 4ed339b2fc2a0ffe590bbbb058816e86eae71062 Mon Sep 17 00:00:00 2001 From: Mukesh Dewangan <143334238+MukeshK17@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:08:46 +0530 Subject: [PATCH 01/17] fix(vertex_ai): map missing image on Gemini image edit to BadRequestError (400) (#50244) --- .../vertex_ai/image_edit/vertex_gemini_transformation.py | 6 +++++- .../image_edit/test_vertex_ai_image_edit_transformation.py | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py index 0b90f74ea6d..3633bb68b50 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -166,7 +166,11 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): ) -> tuple[dict[str, object], RequestFiles | None]: inline_parts: Final = self._prepare_inline_image_parts(image) if image else [] if not inline_parts: - raise ValueError("Vertex AI Gemini image edit requires at least one image.") + raise litellm.BadRequestError( + message="Vertex AI Gemini image edit requires at least one image.", + model=model, + llm_provider="vertex_ai", + ) # Build parts list with image and prompt (if provided) text_parts: Final[list[HttpxPartType]] = [{"text": prompt}] if prompt is not None and prompt != "" else [] diff --git a/tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py b/tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py index 60d7629276f..f36480ae7df 100644 --- a/tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py +++ b/tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py @@ -231,10 +231,10 @@ class TestVertexAIGeminiImageEditTransformation: ) def test_transform_image_edit_request_without_image_raises(self) -> None: - """Test that missing image raises ValueError""" + """Test that missing image raises BadRequestError with 400 status code""" optional_params = {} - with pytest.raises(ValueError, match="requires at least one image"): + with pytest.raises(litellm.BadRequestError, match="requires at least one image") as exc_info: self.config.transform_image_edit_request( model=self.model, prompt=self.prompt, @@ -243,6 +243,7 @@ class TestVertexAIGeminiImageEditTransformation: litellm_params=MagicMock(), headers={}, ) + assert exc_info.value.status_code == 400 def test_validate_environment_with_litellm_params(self) -> None: """Test validate_environment uses credentials from litellm_params""" From 1f98627d4a42620e3fc94495b39082d841c2c5f1 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:50:01 -0800 Subject: [PATCH 02/17] refactor: declare each lazy litellm attribute once and resolve them through one function (#50523) --- litellm/__init__.py | 253 +-- litellm/_lazy_imports.py | 483 +---- litellm/_lazy_imports_registry.py | 2312 ++++++---------------- litellm/utils.py | 13 +- tests/unit/test_lazy_imports.py | 413 +--- tests/unit/test_private_usage_aliases.py | 1 - 6 files changed, 816 insertions(+), 2659 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 8e5dd68a186..8a73f3447b8 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -81,7 +81,6 @@ from litellm.constants import ( import httpx from openai import DefaultAsyncHttpxClient, DefaultHttpxClient -# register_async_client_cleanup is lazy-loaded and called on first access litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" @@ -98,7 +97,6 @@ failure_callback: List[CALLBACK_TYPES] = [] service_callback: List[CALLBACK_TYPES] = [] audit_log_callbacks: List[CALLBACK_TYPES] = [] batch_line_item_callbacks: Sequence[CALLBACK_TYPES] = () -# logging_callback_manager is lazy-loaded via __getattr__ _custom_logger_compatible_callbacks_literal = Literal[ "lago", "openmeter", @@ -507,8 +505,6 @@ skill_search_embedding_model: Optional[str] = None public_model_groups_links: dict[str, str | dict[str, object]] = {} #### REQUEST PRIORITIZATION ####### priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None -# priority_reservation_settings is lazy-loaded via __getattr__ -# Only declare for type checking - at runtime __getattr__ handles it if TYPE_CHECKING: priority_reservation_settings: Optional["PriorityReservationSettings"] = None @@ -543,8 +539,6 @@ _google_kms_resource_name: Optional[str] = None _key_management_system: Optional["KeyManagementSystem"] = None # Note: KeyManagementSettings must be eagerly imported because _key_management_settings # is accessed during import time in secret_managers/main.py -# We'll import it after the lazy import system is set up -# We can't define it here because KeyManagementSettings is lazy-loaded #### PII MASKING #### output_parse_pii: bool = False ############################################# @@ -1196,8 +1190,6 @@ model_list = list( model_list_set = set(model_list) -# provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time - def _build_models_by_provider() -> dict: return { @@ -1378,10 +1370,6 @@ all_embedding_models = ( ####### VIDEO GENERATION MODELS ################### openai_video_generation_models = ["sora-2"] -# timeout is lazy-loaded via __getattr__ -# get_llm_provider is lazy-loaded via __getattr__ -# remove_index_from_tool_calls is lazy-loaded via __getattr__ - # Import KeyManagementSettings here (before utils import) because _key_management_settings # is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils) from litellm.types.secret_managers.main import KeyManagementSettings @@ -1391,8 +1379,6 @@ _key_management_settings: KeyManagementSettings = KeyManagementSettings() # client must be imported immediately as it's used as a decorator at function definition time from .utils import client -# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py -# (which imports tiktoken) at import time from .llms.custom_llm import CustomLLM from .llms.anthropic.common_utils import AnthropicModelInfo @@ -1416,12 +1402,9 @@ from .llms.bedrock.embed.amazon_titan_v2_transformation import ( ) from .llms.topaz.common_utils import TopazModelInfo -# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access -# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) -# All remaining configs are now lazy loaded - see _lazy_imports_registry.py # Import LlmProviders here (before main import) because it's imported during import time # in multiple places including openai.py (via main import) @@ -1650,8 +1633,6 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None: global_gitlab_config = config -# Lazy loading system for heavy modules to reduce initial import time and memory usage - if TYPE_CHECKING: from . import guardrails as guardrails from litellm.types.utils import ModelInfo as _ModelInfoType @@ -1659,7 +1640,6 @@ if TYPE_CHECKING: from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.caching.caching import Cache - # Type stubs for lazy-loaded configs to help mypy from .llms.bedrock.chat.converse_transformation import ( AmazonConverseConfig as AmazonConverseConfig, ) @@ -2088,7 +2068,6 @@ if TYPE_CHECKING: ) from .llms.gdc.chat.transformation import GDCGeminiConfig as GDCGeminiConfig - # Type stubs for lazy-loaded config instances openaiOSeriesConfig: OpenAIOSeriesConfig openAIGPTConfig: OpenAIGPTConfig openAIGPTAudioConfig: OpenAIGPTAudioConfig @@ -2146,7 +2125,6 @@ if TYPE_CHECKING: VertexGeminiConfig as _VertexGeminiConfig, ) - # Type stubs for lazy-loaded config classes (to help mypy understand types) VLLMConfig: Type[_VLLMConfig] DeepSeekChatConfig: Type[_DeepSeekChatConfig] TencentChatConfig: Type[_TencentChatConfig] @@ -2423,63 +2401,34 @@ if TYPE_CHECKING: module_level_aclient: AsyncHTTPHandler module_level_client: HTTPHandler - # Bedrock tool name mappings instance (lazy-loaded) from litellm.caching.caching import InMemoryCache bedrock_tool_name_mappings: InMemoryCache - # Azure exception class (lazy-loaded) from litellm.llms.azure.common_utils import AzureOpenAIError - # Secret manager types (lazy-loaded) from litellm.types.secret_managers.main import ( KeyManagementSystem, KeyManagementSettings, # Not lazy-loaded - needed for _key_management_settings initialization ) - # Custom logger class (lazy-loaded) from litellm.integrations.custom_logger import CustomLogger - # Datadog LLM observability params (lazy-loaded) from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams - # Logging callback manager class and instance (lazy-loaded) from litellm.litellm_core_utils.logging_callback_manager import ( LoggingCallbackManager, ) logging_callback_manager: LoggingCallbackManager - # provider_list is lazy-loaded from litellm.types.utils import LlmProviders provider_list: List[Union[LlmProviders, str]] - # Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block - -# Track if async client cleanup has been registered (for lazy loading) _async_client_cleanup_registered = False -_AGENT_EXPORTS: Final = frozenset( - { - "agent", - "aagent", - "agent_session", - "aagent_session", - "agent_resume", - "aagent_resume", - "agent_capabilities", - "Harness", - "ClaudeCodeOptions", - "CodexOptions", - "OpenCodeOptions", - "PiOptions", - "DeepAgentsOptions", - "ToolLoopOptions", - } -) - # Eager loading for backwards compatibility with VCR and other HTTP recording tools # When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time # For now, this only affects encoding (tiktoken) as it was the only reported issue @@ -2492,9 +2441,7 @@ if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", def __getattr__(name: str) -> Any: - """Lazy import handler with cached registry for improved performance.""" global _async_client_cleanup_registered - # Register async client cleanup on first access (only once) if not _async_client_cleanup_registered: from litellm.llms.custom_httpx.async_client_cleanup import ( register_async_client_cleanup, @@ -2503,205 +2450,9 @@ def __getattr__(name: str) -> Any: register_async_client_cleanup() _async_client_cleanup_registered = True - # Use cached registry from _lazy_imports instead of importing tuples every time - from ._lazy_imports import get_lazy_import_registry + from ._lazy_imports import resolve_lazy_attribute - registry: Final = get_lazy_import_registry() + return resolve_lazy_attribute(__name__, name) - # Check if name is in registry and call the cached handler function - if name in registry: - handler_func: Final = registry[name] - return handler_func(name) - - if name in ("proxy", "guardrails"): - import importlib - - return importlib.import_module(f"litellm.{name}") - - if name == "harness" or name in _AGENT_EXPORTS: - import importlib - - harness_module = importlib.import_module("litellm.harness") - return harness_module if name == "harness" else getattr(harness_module, name) - - # Lazy load encoding from main.py to avoid heavy tiktoken import - if name == "encoding": - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - # Check if already cached - if "encoding" not in _globals: - from .main import encoding as _encoding - - _globals["encoding"] = _encoding - return _globals["encoding"] - - # Lazy load bedrock_tool_name_mappings instance - if name == "bedrock_tool_name_mappings": - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - # Check if already cached - if "bedrock_tool_name_mappings" not in _globals: - from .llms.bedrock.chat.invoke_handler import ( - bedrock_tool_name_mappings as _bedrock_tool_name_mappings, - ) - - _globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings - return _globals["bedrock_tool_name_mappings"] - - # Lazy load AzureOpenAIError exception class - if name == "AzureOpenAIError": - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - # Check if already cached - if "AzureOpenAIError" not in _globals: - from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError - - _globals["AzureOpenAIError"] = _AzureOpenAIError - return _globals["AzureOpenAIError"] - - # Lazy load openaiOSeriesConfig instance - if name == "openaiOSeriesConfig": - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - if "openaiOSeriesConfig" not in _globals: - # Import the config class and instantiate it - config_class = __getattr__("OpenAIOSeriesConfig") - _globals["openaiOSeriesConfig"] = config_class() - return _globals["openaiOSeriesConfig"] - - # Lazy load other config instances - _config_instances: Final = { - "openAIGPTConfig": "OpenAIGPTConfig", - "openAIGPTAudioConfig": "OpenAIGPTAudioConfig", - "openAIGPT5Config": "OpenAIGPT5Config", - "nvidiaNimConfig": "NvidiaNimConfig", - "nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig", - } - if name in _config_instances: - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - if name not in _globals: - # Import the config class and instantiate it - config_class = __getattr__(_config_instances[name]) - _globals[name] = config_class() - return _globals[name] - - # Handle OpenAIO1Config alias - if name == "OpenAIO1Config": - return __getattr__("OpenAIOSeriesConfig") - - # Lazy load provider_list - if name == "provider_list": - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - # Check if already cached - if "provider_list" not in _globals: - # LlmProviders is eagerly imported above, so we can import it directly - from litellm.types.utils import LlmProviders - - _globals["provider_list"] = list(LlmProviders) - return _globals["provider_list"] - - # Lazy load priority_reservation_settings instance - if name == "priority_reservation_settings": - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - # Check if already cached - if "priority_reservation_settings" not in _globals: - # Import the class and instantiate it - PriorityReservationSettings: Final = __getattr__("PriorityReservationSettings") - _globals["priority_reservation_settings"] = PriorityReservationSettings() - return _globals["priority_reservation_settings"] - - # Lazy load logging_callback_manager instance - if name == "logging_callback_manager": - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - # Check if already cached - if "logging_callback_manager" not in _globals: - # Import the class and instantiate it - LoggingCallbackManager: Final = __getattr__("LoggingCallbackManager") - _globals["logging_callback_manager"] = LoggingCallbackManager() - return _globals["logging_callback_manager"] - - # Lazy load _service_logger module - if name == "_service_logger": - from ._lazy_imports import get_litellm_globals - - _globals = get_litellm_globals() - # Check if already cached - if "_service_logger" not in _globals: - # Import the module lazily - import litellm._service_logger - - _globals["_service_logger"] = litellm._service_logger - return _globals["_service_logger"] - - # Lazy load evals module functions - if name in [ - "acreate_eval", - "alist_evals", - "aget_eval", - "aupdate_eval", - "adelete_eval", - "acancel_eval", - "create_eval", - "list_evals", - "get_eval", - "update_eval", - "delete_eval", - "cancel_eval", - "acreate_run", - "alist_runs", - "aget_run", - "acancel_run", - "adelete_run", - "create_run", - "list_runs", - "get_run", - "cancel_run", - "delete_run", - ]: - from litellm.evals.main import ( - acreate_eval, - alist_evals, - aget_eval, - aupdate_eval, - adelete_eval, - acancel_eval, - create_eval, - list_evals, - get_eval, - update_eval, - delete_eval, - cancel_eval, - acreate_run, - alist_runs, - aget_run, - acancel_run, - adelete_run, - create_run, - list_runs, - get_run, - cancel_run, - delete_run, - ) - from types import MappingProxyType - - evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals()) - return evals_exports[name] - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -# ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time mark_litellm_import_complete() diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 1a7dc343745..1ea4bb52668 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -1,60 +1,14 @@ -""" -Lazy Import System - -This module implements lazy loading for LiteLLM attributes. Instead of importing -everything when the module loads, we only import things when they're actually used. - -How it works: -1. When someone accesses `litellm.some_attribute`, Python calls __getattr__ in __init__.py -2. __getattr__ looks up the attribute name in a registry -3. The registry points to a handler function (like _lazy_import_utils) -4. The handler function imports the module and returns the attribute -5. The result is cached so we don't import it again - -This makes importing litellm much faster because we don't load heavy dependencies -until they're actually needed. -""" +"""Resolve the attributes of `litellm` and `litellm.utils` that are imported on first access.""" import importlib import sys from collections.abc import Callable, Mapping -from types import ModuleType -from typing import TYPE_CHECKING, Any, Final +from types import MappingProxyType, ModuleType +from typing import TYPE_CHECKING, Final from typing_extensions import ReadOnly, TypedDict -# Import all the data structures that define what can be lazy-loaded -# These are just lists of names and maps of where to find them -from ._lazy_imports_registry import ( - # Import maps - _BEDROCK_TYPES_IMPORT_MAP, - _CACHING_IMPORT_MAP, - _COST_CALCULATOR_IMPORT_MAP, - _DOTPROMPT_IMPORT_MAP, - _LITELLM_LOGGING_IMPORT_MAP, - _LLM_CONFIGS_IMPORT_MAP, - _LLM_PROVIDER_LOGIC_IMPORT_MAP, - _TOKEN_COUNTER_IMPORT_MAP, - _TYPES_IMPORT_MAP, - _TYPES_UTILS_IMPORT_MAP, - _UTILS_IMPORT_MAP, - _UTILS_MODULE_IMPORT_MAP, - # Name tuples - BEDROCK_TYPES_NAMES, - CACHING_NAMES, - COST_CALCULATOR_NAMES, - DOTPROMPT_NAMES, - HTTP_HANDLER_NAMES, - LITELLM_LOGGING_NAMES, - LLM_CLIENT_CACHE_NAMES, - LLM_CONFIG_NAMES, - LLM_PROVIDER_LOGIC_NAMES, - TOKEN_COUNTER_NAMES, - TYPES_NAMES, - TYPES_UTILS_NAMES, - UTILS_MODULE_NAMES, - UTILS_NAMES, -) +from ._lazy_imports_registry import LITELLM_ATTRIBUTES, UTILS_ATTRIBUTES if TYPE_CHECKING: import httpx @@ -62,35 +16,6 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer -def get_litellm_globals() -> dict[str, object]: - """ - Get the globals dictionary of the litellm module. - - This is where we cache imported attributes so we don't import them twice. - When you do `litellm.some_function`, it gets stored in this dictionary. - """ - return sys.modules["litellm"].__dict__ - - -def _get_utils_globals() -> dict[str, object]: - """ - Get the globals dictionary of the utils module. - - This is where we cache imported attributes so we don't import them twice. - When you do `litellm.utils.some_function`, it gets stored in this dictionary. - """ - return sys.modules["litellm.utils"].__dict__ - - -def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None": - """Read the configured `litellm.request_timeout` used for the module level http clients.""" - return litellm_globals.get("request_timeout") - - -# These are special lazy loaders for things that are used internally -# They're separate from the main lazy import system because they have specific use cases - - def get_default_encoding() -> "Tokenizer": from litellm.rust_bridge.tokenizer import get_encoding @@ -100,138 +25,43 @@ def get_default_encoding() -> "Tokenizer": _get_default_encoding = get_default_encoding -# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time -_get_modified_max_tokens_func: "Callable[..., int | None] | None" = None - - def _get_modified_max_tokens() -> "Callable[..., int | None]": - """ - Lazily load and cache the get_modified_max_tokens function. + from litellm.litellm_core_utils.token_counter import get_modified_max_tokens - This avoids importing `litellm.litellm_core_utils.token_counter` at `litellm` import time. - The function is cached after the first import. - - This is used internally by utils.py functions that need the token counter but shouldn't - trigger its import during module load. - """ - global _get_modified_max_tokens_func - if _get_modified_max_tokens_func is None: - from litellm.litellm_core_utils.token_counter import ( - get_modified_max_tokens as _get_modified_max_tokens_imported, - ) - - _get_modified_max_tokens_func = _get_modified_max_tokens_imported - return _get_modified_max_tokens_func - - -# Lazy loader for token_counter to avoid importing token_counter module at module import time -_token_counter_new_func: "Callable[..., int] | None" = None -_messages_reach_token_count_func: "Callable[..., bool] | None" = None + return get_modified_max_tokens def get_token_counter_new() -> "Callable[..., int]": - """ - Lazily load and cache the token_counter function (aliased as token_counter_new). + from litellm.litellm_core_utils.token_counter import token_counter - This avoids importing `litellm.litellm_core_utils.token_counter` at `litellm` import time. - The function is cached after the first import. - - This is used internally by utils.py functions that need the token counter but shouldn't - trigger its import during module load. - """ - global _token_counter_new_func - if _token_counter_new_func is None: - from litellm.litellm_core_utils.token_counter import ( - token_counter as _token_counter_imported, - ) - - _token_counter_new_func = _token_counter_imported - return _token_counter_new_func + return token_counter _get_token_counter_new = get_token_counter_new def get_messages_reach_token_count() -> "Callable[..., bool]": - """Lazily load ``messages_reach_token_count`` for the same reason as ``get_token_counter_new``.""" - global _messages_reach_token_count_func - if _messages_reach_token_count_func is None: - from litellm.litellm_core_utils.token_counter import ( - messages_reach_token_count as _messages_reach_token_count_imported, - ) + from litellm.litellm_core_utils.token_counter import messages_reach_token_count - _messages_reach_token_count_func = _messages_reach_token_count_imported - return _messages_reach_token_count_func + return messages_reach_token_count _get_messages_reach_token_count = get_messages_reach_token_count -# ============================================================================ -# MAIN LAZY IMPORT SYSTEM -# ============================================================================ - -# This registry maps attribute names (like "ModelResponse") to handler functions -# It's built once the first time someone accesses a lazy-loaded attribute -# Example: {"ModelResponse": _lazy_import_utils, "Cache": _lazy_import_caching, ...} -_LAZY_IMPORT_REGISTRY: dict[str, Callable[[str], object]] | None = None +_NOT_LOADED: Final = object() -def get_lazy_import_registry() -> dict[str, Callable[[str], object]]: - """ - Build the registry that maps attribute names to their handler functions. - - This is called once, the first time someone accesses a lazy-loaded attribute. - After that, we just look up the handler function in this dictionary. - - Returns: - Dictionary like {"ModelResponse": _lazy_import_utils, ...} - """ - global _LAZY_IMPORT_REGISTRY - if _LAZY_IMPORT_REGISTRY is None: - # Build the registry by going through each category and mapping - # all the names in that category to their handler function - _LAZY_IMPORT_REGISTRY = {} - # For each category, map all its names to the handler function - # Example: All names in UTILS_NAMES get mapped to _lazy_import_utils - for name in COST_CALCULATOR_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_cost_calculator - for name in LITELLM_LOGGING_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_litellm_logging - for name in UTILS_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils - for name in TOKEN_COUNTER_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_token_counter - for name in LLM_CLIENT_CACHE_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_client_cache - for name in BEDROCK_TYPES_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_bedrock_types - for name in TYPES_UTILS_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_types_utils - for name in CACHING_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_caching - for name in HTTP_HANDLER_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_http_handlers - for name in DOTPROMPT_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_dotprompt - for name in LLM_CONFIG_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_configs - for name in TYPES_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_types - for name in LLM_PROVIDER_LOGIC_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic - for name in UTILS_MODULE_NAMES: - _LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils_module - - return _LAZY_IMPORT_REGISTRY +def _cached(namespace: str, name: str) -> object: + return vars(sys.modules[namespace]).get(name, _NOT_LOADED) -_get_lazy_import_registry = get_lazy_import_registry +def _cache(namespace: str, name: str, value: object) -> object: + setattr(sys.modules[namespace], name, value) + return value class _AttributeView(TypedDict): - """Holds one module attribute so the lazily fetched value is read back as ``object``.""" - value: ReadOnly[object] @@ -240,241 +70,88 @@ def _module_attribute(module: ModuleType, attr_name: str) -> object: return attribute["value"] -def _generic_lazy_import(name: str, import_map: Mapping[str, tuple[str, str]], category: str) -> object: - """ - Generic function that handles lazy importing for most attributes. +def _import(namespace: str, name: str, location: tuple[str, str]) -> object: + cached: Final = _cached(namespace, name) + if cached is not _NOT_LOADED: + return cached + module_path, attr_name = location + module: Final[ModuleType] = importlib.import_module(module_path, package="litellm") + return _cache(namespace, name, _module_attribute(module, attr_name)) - This is the workhorse function - it does the actual importing and caching. - Most handler functions just call this with their specific import map. - Steps: - 1. Check if the name exists in the import map (if not, raise error) - 2. Check if we've already imported it (if yes, return cached value) - 3. Look up where to find it (module_path and attr_name from the map) - 4. Import the module (Python caches this automatically) - 5. Get the attribute from the module - 6. Cache it in _globals so we don't import again - 7. Return it +def _instance_of(class_name: str) -> Callable[[], object]: + def create() -> object: + cls: Final = resolve_lazy_attribute("litellm", class_name) + assert isinstance(cls, type) + return cls() - Args: - name: The attribute name someone is trying to access (e.g., "ModelResponse") - import_map: Dictionary telling us where to find each attribute - Format: {"ModelResponse": (".utils", "ModelResponse")} - category: Just for error messages (e.g., "Utils", "Cost calculator") - """ - # Step 1: Make sure this attribute exists in our map - if name not in import_map: - raise AttributeError(f"{category} lazy import: unknown attribute {name!r}") + return create - # Step 2: Get the cache (where we store imported things) - _globals: Final = get_litellm_globals() - # Step 3: If we've already imported it, just return the cached version - if name in _globals: - return _globals[name] +def _submodule(module_path: str) -> Callable[[], object]: + return lambda: importlib.import_module(module_path) - # Step 4: Look up where to find this attribute - # The map tells us: (module_path, attribute_name) - # Example: (".utils", "ModelResponse") means "look in .utils module, get ModelResponse" - module_path, attr_name = import_map[name] - # Step 5: Import the module - # Python automatically caches modules in sys.modules, so calling this twice is fast - # If module_path starts with ".", it's a relative import (needs package="litellm") - # Otherwise it's an absolute import (like "litellm.caching.caching") - if module_path.startswith("."): - module = importlib.import_module(module_path, package="litellm") - else: - module = importlib.import_module(module_path) +def _request_timeout() -> "float | httpx.Timeout | None": + import httpx - # Step 6: Get the actual attribute from the module - # Example: getattr(utils_module, "ModelResponse") returns the ModelResponse class - value: Final = _module_attribute(module, attr_name) + timeout: Final = _cached("litellm", "request_timeout") + return timeout if isinstance(timeout, (int, float, httpx.Timeout)) else None - # Step 7: Cache it so we don't have to import again next time - _globals[name] = value - # Step 8: Return it - return value +def _module_level_aclient() -> object: + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + return get_async_httpx_client( + llm_provider="litellm_module_level_client", + params={"timeout": _request_timeout(), "client_alias": "module level aclient"}, + ) -# ============================================================================ -# HANDLER FUNCTIONS -# ============================================================================ -# These functions are called when someone accesses a lazy-loaded attribute. -# Most of them just call _generic_lazy_import with their specific import map. -# The registry (above) maps attribute names to these handler functions. +def _module_level_client() -> object: + from litellm.llms.custom_httpx.http_handler import HTTPHandler -def _lazy_import_utils(name: str) -> object: - """Handler for utils module attributes (ModelResponse, token_counter, etc.)""" - return _generic_lazy_import(name, _UTILS_IMPORT_MAP, "Utils") + return HTTPHandler(timeout=_request_timeout()) -def _lazy_import_cost_calculator(name: str) -> object: - """Handler for cost calculator functions (completion_cost, cost_per_token, etc.)""" - return _generic_lazy_import(name, _COST_CALCULATOR_IMPORT_MAP, "Cost calculator") +def _provider_list() -> object: + from litellm.types.utils import LlmProviders + return list(LlmProviders) -def _lazy_import_token_counter(name: str) -> object: - """Handler for token counter utilities""" - return _generic_lazy_import(name, _TOKEN_COUNTER_IMPORT_MAP, "Token counter") +_LITELLM_FACTORIES: Final[Mapping[str, Callable[[], object]]] = MappingProxyType( + { + "in_memory_llm_clients_cache": _instance_of("LLMClientCache"), + "module_level_aclient": _module_level_aclient, + "module_level_client": _module_level_client, + "openaiOSeriesConfig": _instance_of("OpenAIOSeriesConfig"), + "openAIGPTConfig": _instance_of("OpenAIGPTConfig"), + "openAIGPTAudioConfig": _instance_of("OpenAIGPTAudioConfig"), + "openAIGPT5Config": _instance_of("OpenAIGPT5Config"), + "nvidiaNimConfig": _instance_of("NvidiaNimConfig"), + "nvidiaNimEmbeddingConfig": _instance_of("NvidiaNimEmbeddingConfig"), + "priority_reservation_settings": _instance_of("PriorityReservationSettings"), + "logging_callback_manager": _instance_of("LoggingCallbackManager"), + "provider_list": _provider_list, + "proxy": _submodule("litellm.proxy"), + "guardrails": _submodule("litellm.guardrails"), + "harness": _submodule("litellm.harness"), + "_service_logger": _submodule("litellm._service_logger"), + } +) -def _lazy_import_bedrock_types(name: str) -> object: - """Handler for Bedrock type aliases""" - return _generic_lazy_import(name, _BEDROCK_TYPES_IMPORT_MAP, "Bedrock types") - -def _lazy_import_types_utils(name: str) -> object: - """Handler for types from litellm.types.utils (BudgetConfig, ImageObject, etc.)""" - return _generic_lazy_import(name, _TYPES_UTILS_IMPORT_MAP, "Types utils") - - -def _lazy_import_caching(name: str) -> object: - """Handler for caching classes (Cache, DualCache, RedisCache, etc.)""" - return _generic_lazy_import(name, _CACHING_IMPORT_MAP, "Caching") - - -def _lazy_import_dotprompt(name: str) -> object: - """Handler for dotprompt integration globals""" - return _generic_lazy_import(name, _DOTPROMPT_IMPORT_MAP, "Dotprompt") - - -def _lazy_import_types(name: str) -> object: - """Handler for type classes (GuardrailItem, etc.)""" - return _generic_lazy_import(name, _TYPES_IMPORT_MAP, "Types") - - -def _lazy_import_llm_configs(name: str) -> object: - """Handler for LLM config classes (AnthropicConfig, OpenAILikeChatConfig, etc.)""" - return _generic_lazy_import(name, _LLM_CONFIGS_IMPORT_MAP, "LLM config") - - -def _lazy_import_litellm_logging(name: str) -> object: - """Handler for litellm_logging module (Logging, modify_integration)""" - return _generic_lazy_import(name, _LITELLM_LOGGING_IMPORT_MAP, "Litellm logging") - - -def _lazy_import_llm_provider_logic(name: str) -> object: - """Handler for LLM provider logic functions (get_llm_provider, etc.)""" - return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic") - - -def _lazy_import_utils_module(name: str) -> object: - """ - Handler for utils module lazy imports. - - This uses a custom implementation because utils module needs to use - _get_utils_globals() instead of get_litellm_globals() for caching. - """ - # Check if this attribute exists in our map - if name not in _UTILS_MODULE_IMPORT_MAP: - raise AttributeError(f"Utils module lazy import: unknown attribute {name!r}") - - # Get the cache (where we store imported things) - use utils globals - _globals: Final = _get_utils_globals() - - # If we've already imported it, just return the cached version - if name in _globals: - return _globals[name] - - # Look up where to find this attribute - module_path, attr_name = _UTILS_MODULE_IMPORT_MAP[name] - - # Import the module - if module_path.startswith("."): - module = importlib.import_module(module_path, package="litellm") - else: - module = importlib.import_module(module_path) - - # Get the actual attribute from the module - value: Final = _module_attribute(module, attr_name) - - # Cache it so we don't have to import again next time - _globals[name] = value - - # Return it - return value - - -# ============================================================================ -# SPECIAL HANDLERS -# ============================================================================ -# These handlers have custom logic that doesn't fit the generic pattern - - -def _lazy_import_llm_client_cache(name: str) -> object: - """ - Handler for LLM client cache - has special logic for singleton instance. - - This one is different because: - - "LLMClientCache" is the class itself - - "in_memory_llm_clients_cache" is a singleton instance of that class - So we need custom logic to handle both cases. - """ - _globals: Final = get_litellm_globals() - - # If already cached, return it - if name in _globals: - return _globals[name] - - # Import the class - from litellm.caching.llm_caching_handler import LLMClientCache - - # If they want the class itself, return it - if name == "LLMClientCache": - _globals["LLMClientCache"] = LLMClientCache - return LLMClientCache - - # If they want the singleton instance, create it (only once) - if name == "in_memory_llm_clients_cache": - instance: Final = LLMClientCache() - _globals["in_memory_llm_clients_cache"] = instance - return instance - - raise AttributeError(f"LLM client cache lazy import: unknown attribute {name!r}") - - -def _lazy_import_http_handlers(name: str) -> object: - """ - Handler for HTTP clients - has special logic for creating client instances. - - This one is different because: - - These aren't just imports, they're actual client instances that need to be created - - They need configuration (timeout, etc.) from the module globals - - They use factory functions instead of direct instantiation - """ - _globals: Final = get_litellm_globals() - - if name == "module_level_aclient": - # Create an async HTTP client using the factory function - from litellm.llms.custom_httpx.http_handler import get_async_httpx_client - - # Get timeout from module config (if set) - async_timeout: Final = _get_module_level_client_timeout(_globals) - params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"} - - # Create the client instance - provider_id: Final = "litellm_module_level_client" - async_client: Final = get_async_httpx_client( - llm_provider=provider_id, - params=params, - ) - - # Cache it so we don't create it again - _globals["module_level_aclient"] = async_client - return async_client - - if name == "module_level_client": - # Create a sync HTTP client - from litellm.llms.custom_httpx.http_handler import HTTPHandler - - sync_timeout: Final = _get_module_level_client_timeout(_globals) - sync_client: Final = HTTPHandler(timeout=sync_timeout) - - # Cache it - _globals["module_level_client"] = sync_client - return sync_client - - raise AttributeError(f"HTTP handlers lazy import: unknown attribute {name!r}") +def resolve_lazy_attribute(module_name: str, name: str) -> object: + """Import, cache and return `name`, the way `module_name.__getattr__` would.""" + litellm_location: Final = LITELLM_ATTRIBUTES.get(name) + if litellm_location is not None: + return _import("litellm", name, litellm_location) + utils_location: Final = UTILS_ATTRIBUTES.get(name) + if utils_location is not None: + return _import("litellm.utils", name, utils_location) + factory: Final = _LITELLM_FACTORIES.get(name) if module_name == "litellm" else None + if factory is None: + raise AttributeError(f"module {module_name!r} has no attribute {name!r}") + cached: Final = _cached("litellm", name) + return cached if cached is not _NOT_LOADED else _cache("litellm", name, factory()) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index bf903e23bc0..1cd82200087 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -1,1658 +1,666 @@ -""" -Registry data for lazy imports. - -This module contains all the name tuples and import maps used by the lazy import system. -Separated from the handler functions for better organization. -""" +"""Where each lazily loaded attribute of `litellm` and `litellm.utils` lives, as name -> (module, attribute).""" +from collections.abc import Mapping +from types import MappingProxyType from typing import Final -# Cost calculator names that support lazy loading via _lazy_import_cost_calculator -COST_CALCULATOR_NAMES: Final = ( - "completion_cost", - "cost_per_token", - "response_cost_calculator", +LITELLM_ATTRIBUTES: Final[Mapping[str, tuple[str, str]]] = MappingProxyType( + { + "completion_cost": (".cost_calculator", "completion_cost"), + "cost_per_token": (".cost_calculator", "cost_per_token"), + "response_cost_calculator": (".cost_calculator", "response_cost_calculator"), + "Logging": ("litellm.litellm_core_utils.litellm_logging", "Logging"), + "modify_integration": ("litellm.litellm_core_utils.litellm_logging", "modify_integration"), + "run_server": ("litellm.proxy.proxy_cli", "run_server"), + "get_optional_params": (".utils", "get_optional_params"), + "get_response_string": (".utils", "get_response_string"), + "token_counter": (".utils", "token_counter"), + "create_pretrained_tokenizer": (".utils", "create_pretrained_tokenizer"), + "create_tokenizer": (".utils", "create_tokenizer"), + "supports_function_calling": (".utils", "supports_function_calling"), + "supports_web_search": (".utils", "supports_web_search"), + "supports_tool_search": (".utils", "supports_tool_search"), + "supports_url_context": (".utils", "supports_url_context"), + "supports_continuation": (".utils", "supports_continuation"), + "supports_response_schema": (".utils", "supports_response_schema"), + "supports_parallel_function_calling": (".utils", "supports_parallel_function_calling"), + "supports_vision": (".utils", "supports_vision"), + "supports_audio_input": (".utils", "supports_audio_input"), + "supports_video_input": (".utils", "supports_video_input"), + "supports_box_questions": (".utils", "supports_box_questions"), + "supports_audio_output": (".utils", "supports_audio_output"), + "supports_system_messages": (".utils", "supports_system_messages"), + "supports_reasoning": (".utils", "supports_reasoning"), + "acreate": (".utils", "acreate"), + "get_max_tokens": (".utils", "get_max_tokens"), + "get_model_info": (".utils", "get_model_info"), + "register_prompt_template": (".utils", "register_prompt_template"), + "validate_environment": (".utils", "validate_environment"), + "check_valid_key": (".utils", "check_valid_key"), + "register_model": (".utils", "register_model"), + "encode": (".utils", "encode"), + "decode": (".utils", "decode"), + "calculate_retry_after": (".utils", "calculate_retry_after"), + "_calculate_retry_after": (".utils", "calculate_retry_after"), + "should_retry": (".utils", "should_retry"), + "_should_retry": (".utils", "should_retry"), + "get_first_chars_messages": (".utils", "get_first_chars_messages"), + "ModelResponse": (".utils", "ModelResponse"), + "ModelResponseStream": (".utils", "ModelResponseStream"), + "EmbeddingResponse": (".utils", "EmbeddingResponse"), + "ImageResponse": (".utils", "ImageResponse"), + "TranscriptionResponse": (".utils", "TranscriptionResponse"), + "TextCompletionResponse": (".utils", "TextCompletionResponse"), + "get_provider_fields": (".utils", "get_provider_fields"), + "ModelResponseListIterator": (".utils", "ModelResponseListIterator"), + "get_valid_models": (".utils", "get_valid_models"), + "timeout": (".timeout", "timeout"), + "get_modified_max_tokens": ("litellm.litellm_core_utils.token_counter", "get_modified_max_tokens"), + "COHERE_EMBEDDING_INPUT_TYPES": ("litellm.types.llms.bedrock", "COHERE_EMBEDDING_INPUT_TYPES"), + "ImageObject": (".types.utils", "ImageObject"), + "BudgetConfig": (".types.utils", "BudgetConfig"), + "all_litellm_params": (".types.utils", "all_litellm_params"), + "_litellm_completion_params": (".types.utils", "all_litellm_params"), + "CredentialItem": (".types.utils", "CredentialItem"), + "PriorityReservationDict": (".types.utils", "PriorityReservationDict"), + "StandardKeyGenerationConfig": (".types.utils", "StandardKeyGenerationConfig"), + "SearchProviders": (".types.utils", "SearchProviders"), + "GenericStreamingChunk": (".types.utils", "GenericStreamingChunk"), + "Cache": ("litellm.caching.caching", "Cache"), + "DualCache": ("litellm.caching.caching", "DualCache"), + "RedisCache": ("litellm.caching.caching", "RedisCache"), + "InMemoryCache": ("litellm.caching.caching", "InMemoryCache"), + "global_prompt_manager": ("litellm.integrations.dotprompt", "global_prompt_manager"), + "global_prompt_directory": ("litellm.integrations.dotprompt", "global_prompt_directory"), + "set_global_prompt_directory": ("litellm.integrations.dotprompt", "set_global_prompt_directory"), + "AmazonConverseConfig": (".llms.bedrock.chat.converse_transformation", "AmazonConverseConfig"), + "OpenAILikeChatConfig": (".llms.openai_like.chat.handler", "OpenAILikeChatConfig"), + "GaladrielChatConfig": (".llms.galadriel.chat.transformation", "GaladrielChatConfig"), + "GithubChatConfig": (".llms.github.chat.transformation", "GithubChatConfig"), + "AzureAnthropicConfig": (".llms.azure_ai.anthropic.transformation", "AzureAnthropicConfig"), + "BytezChatConfig": (".llms.bytez.chat.transformation", "BytezChatConfig"), + "CompactifAIChatConfig": (".llms.compactifai.chat.transformation", "CompactifAIChatConfig"), + "EmpowerChatConfig": (".llms.empower.chat.transformation", "EmpowerChatConfig"), + "MinimaxChatConfig": (".llms.minimax.chat.transformation", "MinimaxChatConfig"), + "AiohttpOpenAIChatConfig": (".llms.aiohttp_openai.chat.transformation", "AiohttpOpenAIChatConfig"), + "HuggingFaceChatConfig": (".llms.huggingface.chat.transformation", "HuggingFaceChatConfig"), + "HuggingFaceEmbeddingConfig": (".llms.huggingface.embedding.transformation", "HuggingFaceEmbeddingConfig"), + "OobaboogaConfig": (".llms.oobabooga.chat.transformation", "OobaboogaConfig"), + "MaritalkConfig": (".llms.maritalk", "MaritalkConfig"), + "OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"), + "DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"), + "AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"), + "BedrockClaudePlatformConfig": (".llms.bedrock.claude_platform.transformation", "BedrockClaudePlatformConfig"), + "AnthropicTextConfig": (".llms.anthropic.completion.transformation", "AnthropicTextConfig"), + "GroqSTTConfig": (".llms.groq.stt.transformation", "GroqSTTConfig"), + "TritonConfig": (".llms.triton.completion.transformation", "TritonConfig"), + "TritonGenerateConfig": (".llms.triton.completion.transformation", "TritonGenerateConfig"), + "TritonInferConfig": (".llms.triton.completion.transformation", "TritonInferConfig"), + "TritonEmbeddingConfig": (".llms.triton.embedding.transformation", "TritonEmbeddingConfig"), + "HuggingFaceRerankConfig": (".llms.huggingface.rerank.transformation", "HuggingFaceRerankConfig"), + "DatabricksConfig": (".llms.databricks.chat.transformation", "DatabricksConfig"), + "DatabricksEmbeddingConfig": (".llms.databricks.embed.transformation", "DatabricksEmbeddingConfig"), + "PredibaseConfig": (".llms.predibase.chat.transformation", "PredibaseConfig"), + "ReplicateConfig": (".llms.replicate.chat.transformation", "ReplicateConfig"), + "SnowflakeConfig": (".llms.snowflake.chat.transformation", "SnowflakeConfig"), + "CohereRerankConfig": (".llms.cohere.rerank.transformation", "CohereRerankConfig"), + "CohereRerankV2Config": (".llms.cohere.rerank_v2.transformation", "CohereRerankV2Config"), + "AzureAIRerankConfig": (".llms.azure_ai.rerank.transformation", "AzureAIRerankConfig"), + "InfinityRerankConfig": (".llms.infinity.rerank.transformation", "InfinityRerankConfig"), + "JinaAIRerankConfig": (".llms.jina_ai.rerank.transformation", "JinaAIRerankConfig"), + "ScalewayRerankConfig": (".llms.scaleway.rerank.transformation", "ScalewayRerankConfig"), + "DeepinfraRerankConfig": (".llms.deepinfra.rerank.transformation", "DeepinfraRerankConfig"), + "HostedVLLMRerankConfig": (".llms.hosted_vllm.rerank.transformation", "HostedVLLMRerankConfig"), + "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), + "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"), + "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), + "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"), + "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), + "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"), + "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), + "CloudflareExtractionConfig": (".llms.cloudflare.extraction.transformation", "CloudflareExtractionConfig"), + "StrandsDeciderDecisionsConfig": ( + ".llms.strands_decider.decisions.transformation", + "StrandsDeciderDecisionsConfig", + ), + "DatabricksDecisionsConfig": (".llms.databricks.decisions.transformation", "DatabricksDecisionsConfig"), + "HostedVLLMDecisionsConfig": (".llms.hosted_vllm.decisions.transformation", "HostedVLLMDecisionsConfig"), + "LMStudioDecisionsConfig": (".llms.lm_studio.decisions.transformation", "LMStudioDecisionsConfig"), + "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"), + "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), + "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), + "EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"), + "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), + "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), + "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), + "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), + "NvidiaNimRerankConfig": (".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig"), + "NvidiaNimRankingConfig": (".llms.nvidia_nim.rerank.ranking_transformation", "NvidiaNimRankingConfig"), + "VertexAIRerankConfig": (".llms.vertex_ai.rerank.transformation", "VertexAIRerankConfig"), + "FireworksAIRerankConfig": (".llms.fireworks_ai.rerank.transformation", "FireworksAIRerankConfig"), + "VoyageRerankConfig": (".llms.voyage.rerank.transformation", "VoyageRerankConfig"), + "IBMWatsonXRerankConfig": (".llms.watsonx.rerank.transformation", "IBMWatsonXRerankConfig"), + "ClarifaiConfig": (".llms.clarifai.chat.transformation", "ClarifaiConfig"), + "AI21ChatConfig": (".llms.ai21.chat.transformation", "AI21ChatConfig"), + "LlamaAPIConfig": (".llms.meta_llama.chat.transformation", "LlamaAPIConfig"), + "TogetherAITextCompletionConfig": ( + ".llms.together_ai.completion.transformation", + "TogetherAITextCompletionConfig", + ), + "CloudflareChatConfig": (".llms.cloudflare.chat.transformation", "CloudflareChatConfig"), + "NovitaConfig": (".llms.novita.chat.transformation", "NovitaConfig"), + "PetalsConfig": (".llms.petals.completion.transformation", "PetalsConfig"), + "OllamaChatConfig": (".llms.ollama.chat.transformation", "OllamaChatConfig"), + "OllamaConfig": (".llms.ollama.completion.transformation", "OllamaConfig"), + "SagemakerConfig": (".llms.sagemaker.completion.transformation", "SagemakerConfig"), + "SagemakerChatConfig": (".llms.sagemaker.chat.transformation", "SagemakerChatConfig"), + "SagemakerNovaConfig": (".llms.sagemaker.nova.transformation", "SagemakerNovaConfig"), + "CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"), + "AnthropicMessagesConfig": (".llms.anthropic.messages.native.transformation", "AnthropicMessagesConfig"), + "BedrockClaudePlatformMessagesConfig": ( + ".llms.bedrock.claude_platform.messages_transformation", + "BedrockClaudePlatformMessagesConfig", + ), + "AmazonAnthropicClaudeMessagesConfig": ( + ".llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation", + "AmazonAnthropicClaudeMessagesConfig", + ), + "AmazonMantleMessagesConfig": (".llms.bedrock.messages.mantle_transformation", "AmazonMantleMessagesConfig"), + "BedrockMantleAnthropicMessagesConfig": ( + ".llms.bedrock_mantle.messages.transformation", + "BedrockMantleAnthropicMessagesConfig", + ), + "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"), + "TogetherAIChatConfig": (".llms.together_ai.chat.transformation", "TogetherAIChatConfig"), + "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"), + "VertexGeminiConfig": (".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", "VertexGeminiConfig"), + "GoogleAIStudioGeminiConfig": (".llms.gemini.chat.transformation", "GoogleAIStudioGeminiConfig"), + "VertexAIAnthropicConfig": ( + ".llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation", + "VertexAIAnthropicConfig", + ), + "VertexAILlama3Config": ( + ".llms.vertex_ai.vertex_ai_partner_models.llama.transformation", + "VertexAILlama3Config", + ), + "VertexAIAi21Config": (".llms.vertex_ai.vertex_ai_partner_models.ai21.transformation", "VertexAIAi21Config"), + "VertexAIMistralConfig": ( + ".llms.vertex_ai.vertex_ai_partner_models.mistral.transformation", + "VertexAIMistralConfig", + ), + "AmazonCohereChatConfig": (".llms.bedrock.chat.invoke_handler", "AmazonCohereChatConfig"), + "AmazonBedrockGlobalConfig": (".llms.bedrock.common_utils", "AmazonBedrockGlobalConfig"), + "AmazonAI21Config": ( + ".llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation", + "AmazonAI21Config", + ), + "AmazonInvokeNovaConfig": ( + ".llms.bedrock.chat.invoke_transformations.amazon_nova_transformation", + "AmazonInvokeNovaConfig", + ), + "AmazonQwen2Config": ( + ".llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation", + "AmazonQwen2Config", + ), + "AmazonQwen3Config": ( + ".llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation", + "AmazonQwen3Config", + ), + "VertexAIConfig": (".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", "VertexGeminiConfig"), + "GeminiConfig": (".llms.gemini.chat.transformation", "GoogleAIStudioGeminiConfig"), + "AmazonAnthropicConfig": ( + ".llms.bedrock.chat.invoke_transformations.anthropic_legacy_text_completions_transformation", + "AmazonAnthropicConfig", + ), + "AmazonAnthropicClaudeConfig": ( + ".llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation", + "AmazonAnthropicClaudeConfig", + ), + "AmazonCohereConfig": ( + ".llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation", + "AmazonCohereConfig", + ), + "AmazonLlamaConfig": ( + ".llms.bedrock.chat.invoke_transformations.amazon_llama_transformation", + "AmazonLlamaConfig", + ), + "AmazonDeepSeekR1Config": ( + ".llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation", + "AmazonDeepSeekR1Config", + ), + "AmazonMistralConfig": ( + ".llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation", + "AmazonMistralConfig", + ), + "AmazonMoonshotConfig": ( + ".llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation", + "AmazonMoonshotConfig", + ), + "AmazonTitanConfig": ( + ".llms.bedrock.chat.invoke_transformations.amazon_titan_transformation", + "AmazonTitanConfig", + ), + "AmazonTwelveLabsPegasusConfig": ( + ".llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation", + "AmazonTwelveLabsPegasusConfig", + ), + "AmazonInvokeConfig": ( + ".llms.bedrock.chat.invoke_transformations.base_invoke_transformation", + "AmazonInvokeConfig", + ), + "AmazonBedrockOpenAIConfig": ( + ".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation", + "AmazonBedrockOpenAIConfig", + ), + "AmazonBedrockRuntimeChatCompletionsConfig": ( + ".llms.bedrock.chat.chat_completions.transformation", + "AmazonBedrockRuntimeChatCompletionsConfig", + ), + "AmazonStabilityConfig": ( + ".llms.bedrock.image_generation.amazon_stability1_transformation", + "AmazonStabilityConfig", + ), + "AmazonStability3Config": ( + ".llms.bedrock.image_generation.amazon_stability3_transformation", + "AmazonStability3Config", + ), + "AmazonNovaCanvasConfig": ( + ".llms.bedrock.image_generation.amazon_nova_canvas_transformation", + "AmazonNovaCanvasConfig", + ), + "AmazonTitanG1Config": (".llms.bedrock.embed.amazon_titan_g1_transformation", "AmazonTitanG1Config"), + "AmazonTitanMultimodalEmbeddingG1Config": ( + ".llms.bedrock.embed.amazon_titan_multimodal_transformation", + "AmazonTitanMultimodalEmbeddingG1Config", + ), + "CohereV2ChatConfig": (".llms.cohere.chat.v2_transformation", "CohereV2ChatConfig"), + "BedrockCohereEmbeddingConfig": (".llms.bedrock.embed.cohere_transformation", "BedrockCohereEmbeddingConfig"), + "TwelveLabsMarengoEmbeddingConfig": ( + ".llms.bedrock.embed.twelvelabs_marengo_transformation", + "TwelveLabsMarengoEmbeddingConfig", + ), + "AmazonNovaEmbeddingConfig": (".llms.bedrock.embed.amazon_nova_transformation", "AmazonNovaEmbeddingConfig"), + "OpenAIConfig": (".llms.openai.openai", "OpenAIConfig"), + "MistralEmbeddingConfig": (".llms.openai.openai", "MistralEmbeddingConfig"), + "OpenAIImageVariationConfig": (".llms.openai.image_variations.transformation", "OpenAIImageVariationConfig"), + "DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"), + "ClfAiGatewayConfig": (".llms.clf_ai_gateway.chat.transformation", "ClfAiGatewayConfig"), + "DeepgramAudioTranscriptionConfig": ( + ".llms.deepgram.audio_transcription.transformation", + "DeepgramAudioTranscriptionConfig", + ), + "TopazImageVariationConfig": (".llms.topaz.image_variations.transformation", "TopazImageVariationConfig"), + "OpenAITextCompletionConfig": ("litellm.llms.openai.completion.transformation", "OpenAITextCompletionConfig"), + "GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"), + "BedrockOpenAIResponsesConfig": (".llms.bedrock.responses.transformation", "BedrockOpenAIResponsesConfig"), + "BedrockMantleChatConfig": (".llms.bedrock_mantle.chat.transformation", "BedrockMantleChatConfig"), + "A2AConfig": (".llms.a2a.chat.transformation", "A2AConfig"), + "GenAIHubOrchestrationConfig": (".llms.sap.chat.transformation", "GenAIHubOrchestrationConfig"), + "VoyageEmbeddingConfig": (".llms.voyage.embedding.transformation", "VoyageEmbeddingConfig"), + "VoyageContextualEmbeddingConfig": ( + ".llms.voyage.embedding.transformation_contextual", + "VoyageContextualEmbeddingConfig", + ), + "VoyageMultimodalEmbeddingConfig": ( + ".llms.voyage.embedding.transformation_multimodal", + "VoyageMultimodalEmbeddingConfig", + ), + "InfinityEmbeddingConfig": (".llms.infinity.embedding.transformation", "InfinityEmbeddingConfig"), + "PerplexityEmbeddingConfig": (".llms.perplexity.embedding.transformation", "PerplexityEmbeddingConfig"), + "AzureAIStudioConfig": (".llms.azure_ai.chat.transformation", "AzureAIStudioConfig"), + "MistralConfig": (".llms.mistral.chat.transformation", "MistralConfig"), + "OpenAIResponsesAPIConfig": (".llms.openai.responses.transformation", "OpenAIResponsesAPIConfig"), + "AzureOpenAIResponsesAPIConfig": (".llms.azure.responses.transformation", "AzureOpenAIResponsesAPIConfig"), + "AzureOpenAIOSeriesResponsesAPIConfig": ( + ".llms.azure.responses.o_series_transformation", + "AzureOpenAIOSeriesResponsesAPIConfig", + ), + "AzureAIResponsesAPIConfig": (".llms.azure_ai.responses.transformation", "AzureAIResponsesAPIConfig"), + "XAIResponsesAPIConfig": (".llms.xai.responses.transformation", "XAIResponsesAPIConfig"), + "LiteLLMProxyResponsesAPIConfig": ( + ".llms.litellm_proxy.responses.transformation", + "LiteLLMProxyResponsesAPIConfig", + ), + "HostedVLLMResponsesAPIConfig": (".llms.hosted_vllm.responses.transformation", "HostedVLLMResponsesAPIConfig"), + "FireworksAIResponsesAPIConfig": ( + ".llms.fireworks_ai.responses.transformation", + "FireworksAIResponsesAPIConfig", + ), + "VolcEngineResponsesAPIConfig": (".llms.volcengine.responses.transformation", "VolcEngineResponsesAPIConfig"), + "ManusResponsesAPIConfig": (".llms.manus.responses.transformation", "ManusResponsesAPIConfig"), + "PerplexityResponsesConfig": (".llms.perplexity.responses.transformation", "PerplexityResponsesConfig"), + "DatabricksResponsesAPIConfig": (".llms.databricks.responses.transformation", "DatabricksResponsesAPIConfig"), + "OpenRouterResponsesAPIConfig": (".llms.openrouter.responses.transformation", "OpenRouterResponsesAPIConfig"), + "BedrockMantleResponsesAPIConfig": ( + ".llms.bedrock_mantle.responses.transformation", + "BedrockMantleResponsesAPIConfig", + ), + "GoogleAIStudioInteractionsConfig": ( + ".llms.gemini.interactions.transformation", + "GoogleAIStudioInteractionsConfig", + ), + "VertexAIInteractionsConfig": (".llms.vertex_ai.interactions.transformation", "VertexAIInteractionsConfig"), + "OpenAIOSeriesConfig": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"), + "AnthropicSkillsConfig": (".llms.anthropic.skills.transformation", "AnthropicSkillsConfig"), + "BaseSkillsAPIConfig": (".llms.base_llm.skills.transformation", "BaseSkillsAPIConfig"), + "GradientAIConfig": (".llms.gradient_ai.chat.transformation", "GradientAIConfig"), + "OpenAIO1Config": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"), + "OpenAIGPTConfig": (".llms.openai.chat.gpt_transformation", "OpenAIGPTConfig"), + "OpenAIGPT5Config": (".llms.openai.chat.gpt_reasoning_transformation", "OpenAIGPT5Config"), + "OpenAIWhisperAudioTranscriptionConfig": ( + ".llms.openai.transcriptions.whisper_transformation", + "OpenAIWhisperAudioTranscriptionConfig", + ), + "OpenAIGPTAudioTranscriptionConfig": ( + ".llms.openai.transcriptions.gpt_transformation", + "OpenAIGPTAudioTranscriptionConfig", + ), + "OpenAIGPTAudioConfig": (".llms.openai.chat.gpt_audio_transformation", "OpenAIGPTAudioConfig"), + "NvidiaNimConfig": (".llms.nvidia_nim.chat.transformation", "NvidiaNimConfig"), + "NvidiaNimEmbeddingConfig": (".llms.nvidia_nim.embed", "NvidiaNimEmbeddingConfig"), + "FeatherlessAIConfig": (".llms.featherless_ai.chat.transformation", "FeatherlessAIConfig"), + "ScaleDownChatConfig": (".llms.scaledown.chat.transformation", "ScaleDownChatConfig"), + "CerebrasConfig": (".llms.cerebras.chat", "CerebrasConfig"), + "NadirConfig": (".llms.nadir.chat.transformation", "NadirConfig"), + "BasetenConfig": (".llms.baseten.chat", "BasetenConfig"), + "SambanovaConfig": (".llms.sambanova.chat", "SambanovaConfig"), + "SambaNovaEmbeddingConfig": (".llms.sambanova.embedding.transformation", "SambaNovaEmbeddingConfig"), + "FireworksAIConfig": (".llms.fireworks_ai.chat.transformation", "FireworksAIConfig"), + "FireworksAITextCompletionConfig": ( + ".llms.fireworks_ai.completion.transformation", + "FireworksAITextCompletionConfig", + ), + "FireworksAIEmbeddingConfig": ( + ".llms.fireworks_ai.embed.fireworks_ai_transformation", + "FireworksAIEmbeddingConfig", + ), + "FriendliaiChatConfig": (".llms.friendliai.chat.transformation", "FriendliaiChatConfig"), + "JinaAIEmbeddingConfig": (".llms.jina_ai.embedding.transformation", "JinaAIEmbeddingConfig"), + "XAIChatConfig": (".llms.xai.chat.transformation", "XAIChatConfig"), + "ZAIChatConfig": (".llms.zai.chat.transformation", "ZAIChatConfig"), + "AIMLChatConfig": (".llms.aiml.chat.transformation", "AIMLChatConfig"), + "VolcEngineChatConfig": (".llms.volcengine.chat.transformation", "VolcEngineChatConfig"), + "CodestralTextCompletionConfig": (".llms.codestral.completion.transformation", "CodestralTextCompletionConfig"), + "InceptionTextCompletionConfig": (".llms.inception.completion.transformation", "InceptionTextCompletionConfig"), + "AzureOpenAIAssistantsAPIConfig": (".llms.azure.azure", "AzureOpenAIAssistantsAPIConfig"), + "HerokuChatConfig": (".llms.heroku.chat.transformation", "HerokuChatConfig"), + "CometAPIConfig": (".llms.cometapi.chat.transformation", "CometAPIConfig"), + "AzureOpenAIConfig": (".llms.azure.chat.gpt_transformation", "AzureOpenAIConfig"), + "AzureOpenAIGPT5Config": (".llms.azure.chat.gpt_reasoning_transformation", "AzureOpenAIGPT5Config"), + "AzureOpenAITextConfig": (".llms.azure.completion.transformation", "AzureOpenAITextConfig"), + "AzureSpeechAudioTranscriptionConfig": ( + ".llms.azure.audio_transcription.transformation", + "AzureSpeechAudioTranscriptionConfig", + ), + "HostedVLLMChatConfig": (".llms.hosted_vllm.chat.transformation", "HostedVLLMChatConfig"), + "HostedVLLMEmbeddingConfig": (".llms.hosted_vllm.embedding.transformation", "HostedVLLMEmbeddingConfig"), + "VolcEngineConfig": (".llms.volcengine.chat.transformation", "VolcEngineChatConfig"), + "LlamafileChatConfig": (".llms.llamafile.chat.transformation", "LlamafileChatConfig"), + "LiteLLMProxyChatConfig": (".llms.litellm_proxy.chat.transformation", "LiteLLMProxyChatConfig"), + "VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"), + "DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"), + "TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"), + "LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"), + "LmStudioEmbeddingConfig": (".llms.lm_studio.embed.transformation", "LmStudioEmbeddingConfig"), + "NscaleConfig": (".llms.nscale.chat.transformation", "NscaleConfig"), + "PerplexityChatConfig": (".llms.perplexity.chat.transformation", "PerplexityChatConfig"), + "AzureOpenAIO1Config": (".llms.azure.chat.o_series_transformation", "AzureOpenAIO1Config"), + "IBMWatsonXAIConfig": (".llms.watsonx.completion.transformation", "IBMWatsonXAIConfig"), + "IBMWatsonXChatConfig": (".llms.watsonx.chat.transformation", "IBMWatsonXChatConfig"), + "IBMWatsonXEmbeddingConfig": (".llms.watsonx.embed.transformation", "IBMWatsonXEmbeddingConfig"), + "GenAIHubEmbeddingConfig": (".llms.sap.embed.transformation", "GenAIHubEmbeddingConfig"), + "IBMWatsonXAudioTranscriptionConfig": ( + ".llms.watsonx.audio_transcription.transformation", + "IBMWatsonXAudioTranscriptionConfig", + ), + "GithubCopilotConfig": (".llms.github_copilot.chat.transformation", "GithubCopilotConfig"), + "GithubCopilotResponsesAPIConfig": ( + ".llms.github_copilot.responses.transformation", + "GithubCopilotResponsesAPIConfig", + ), + "GithubCopilotEmbeddingConfig": ( + ".llms.github_copilot.embedding.transformation", + "GithubCopilotEmbeddingConfig", + ), + "ChatGPTConfig": (".llms.chatgpt.chat.transformation", "ChatGPTConfig"), + "ChatGPTResponsesAPIConfig": (".llms.chatgpt.responses.transformation", "ChatGPTResponsesAPIConfig"), + "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"), + "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"), + "GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"), + "GigaChatEmbeddingConfig": (".llms.gigachat.embedding.transformation", "GigaChatEmbeddingConfig"), + "DashScopeChatConfig": (".llms.dashscope.chat.transformation", "DashScopeChatConfig"), + "QwenCloudChatConfig": (".llms.dashscope.qwencloud", "QwenCloudChatConfig"), + "QwenAIPlatformChatConfig": (".llms.dashscope.qwen_ai_platform", "QwenAIPlatformChatConfig"), + "GDCGeminiConfig": (".llms.gdc.chat.transformation", "GDCGeminiConfig"), + "ModelScopeChatConfig": (".llms.modelscope.chat.transformation", "ModelScopeChatConfig"), + "MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"), + "DockerModelRunnerChatConfig": (".llms.docker_model_runner.chat.transformation", "DockerModelRunnerChatConfig"), + "V0ChatConfig": (".llms.v0.chat.transformation", "V0ChatConfig"), + "OCIChatConfig": (".llms.oci.chat.transformation", "OCIChatConfig"), + "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), + "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), + "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), + "InceptionChatConfig": (".llms.inception.chat.transformation", "InceptionChatConfig"), + "HyperbolicChatConfig": (".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig"), + "VercelAIGatewayConfig": (".llms.vercel_ai_gateway.chat.transformation", "VercelAIGatewayConfig"), + "EdenAIChatConfig": (".llms.edenai.chat.transformation", "EdenAIChatConfig"), + "EdenAIResponsesAPIConfig": (".llms.edenai.responses.transformation", "EdenAIResponsesAPIConfig"), + "EdenAIAnthropicMessagesConfig": (".llms.edenai.messages.transformation", "EdenAIAnthropicMessagesConfig"), + "EdenAIEmbeddingConfig": (".llms.edenai.embedding.transformation", "EdenAIEmbeddingConfig"), + "EdenAIAudioTranscriptionConfig": ( + ".llms.edenai.audio_transcription.transformation", + "EdenAIAudioTranscriptionConfig", + ), + "EdenAITextToSpeechConfig": (".llms.edenai.text_to_speech.transformation", "EdenAITextToSpeechConfig"), + "EdenAIImageGenerationConfig": (".llms.edenai.image_generation.transformation", "EdenAIImageGenerationConfig"), + "EdenAIVideoConfig": (".llms.edenai.videos.transformation", "EdenAIVideoConfig"), + "FalAIChatConfig": (".llms.fal_ai.chat.transformation", "FalAIChatConfig"), + "FalAIError": (".llms.fal_ai.chat.transformation", "FalAIError"), + "OVHCloudChatConfig": (".llms.ovhcloud.chat.transformation", "OVHCloudChatConfig"), + "OVHCloudEmbeddingConfig": (".llms.ovhcloud.embedding.transformation", "OVHCloudEmbeddingConfig"), + "CometAPIEmbeddingConfig": (".llms.cometapi.embed.transformation", "CometAPIEmbeddingConfig"), + "LemonadeChatConfig": (".llms.lemonade.chat.transformation", "LemonadeChatConfig"), + "SnowflakeEmbeddingConfig": (".llms.snowflake.embedding.transformation", "SnowflakeEmbeddingConfig"), + "AmazonNovaChatConfig": (".llms.amazon_nova.chat.transformation", "AmazonNovaChatConfig"), + "SonioxAudioTranscriptionConfig": ( + ".llms.soniox.audio_transcription.transformation", + "SonioxAudioTranscriptionConfig", + ), + "GuardrailItem": ("litellm.types.guardrails", "GuardrailItem"), + "DefaultTeamSSOParams": ("litellm.types.proxy.management_endpoints.ui_sso", "DefaultTeamSSOParams"), + "LiteLLM_UpperboundKeyGenerateParams": ( + "litellm.types.proxy.management_endpoints.ui_sso", + "LiteLLM_UpperboundKeyGenerateParams", + ), + "KeyManagementSystem": ("litellm.types.secret_managers.main", "KeyManagementSystem"), + "PriorityReservationSettings": ("litellm.types.utils", "PriorityReservationSettings"), + "CustomLogger": ("litellm.integrations.custom_logger", "CustomLogger"), + "LoggingCallbackManager": ("litellm.litellm_core_utils.logging_callback_manager", "LoggingCallbackManager"), + "DatadogLLMObsInitParams": ("litellm.types.integrations.datadog_llm_obs", "DatadogLLMObsInitParams"), + "remove_index_from_tool_calls": ("litellm.litellm_core_utils.core_helpers", "remove_index_from_tool_calls"), + "LLMClientCache": (".caching.llm_caching_handler", "LLMClientCache"), + "bedrock_tool_name_mappings": (".llms.bedrock.chat.invoke_handler", "bedrock_tool_name_mappings"), + "AzureOpenAIError": (".llms.azure.common_utils", "AzureOpenAIError"), + "agent": (".harness", "agent"), + "aagent": (".harness", "aagent"), + "agent_session": (".harness", "agent_session"), + "aagent_session": (".harness", "aagent_session"), + "agent_resume": (".harness", "agent_resume"), + "aagent_resume": (".harness", "aagent_resume"), + "agent_capabilities": (".harness", "agent_capabilities"), + "Harness": (".harness", "Harness"), + "ClaudeCodeOptions": (".harness", "ClaudeCodeOptions"), + "CodexOptions": (".harness", "CodexOptions"), + "OpenCodeOptions": (".harness", "OpenCodeOptions"), + "PiOptions": (".harness", "PiOptions"), + "DeepAgentsOptions": (".harness", "DeepAgentsOptions"), + "ToolLoopOptions": (".harness", "ToolLoopOptions"), + "acreate_eval": (".evals.main", "acreate_eval"), + "alist_evals": (".evals.main", "alist_evals"), + "aget_eval": (".evals.main", "aget_eval"), + "aupdate_eval": (".evals.main", "aupdate_eval"), + "adelete_eval": (".evals.main", "adelete_eval"), + "acancel_eval": (".evals.main", "acancel_eval"), + "create_eval": (".evals.main", "create_eval"), + "list_evals": (".evals.main", "list_evals"), + "get_eval": (".evals.main", "get_eval"), + "update_eval": (".evals.main", "update_eval"), + "delete_eval": (".evals.main", "delete_eval"), + "cancel_eval": (".evals.main", "cancel_eval"), + "acreate_run": (".evals.main", "acreate_run"), + "alist_runs": (".evals.main", "alist_runs"), + "aget_run": (".evals.main", "aget_run"), + "acancel_run": (".evals.main", "acancel_run"), + "adelete_run": (".evals.main", "adelete_run"), + "create_run": (".evals.main", "create_run"), + "list_runs": (".evals.main", "list_runs"), + "get_run": (".evals.main", "get_run"), + "cancel_run": (".evals.main", "cancel_run"), + "delete_run": (".evals.main", "delete_run"), + } ) -# Litellm logging names that support lazy loading via _lazy_import_litellm_logging -LITELLM_LOGGING_NAMES: Final = ( - "Logging", - "modify_integration", +UTILS_ATTRIBUTES: Final[Mapping[str, tuple[str, str]]] = MappingProxyType( + { + "encoding": ("litellm.main", "encoding"), + "BaseVectorStore": ("litellm.integrations.vector_store_integrations.base_vector_store", "BaseVectorStore"), + "CredentialAccessor": ("litellm.litellm_core_utils.credential_accessor", "CredentialAccessor"), + "exception_type": ("litellm.litellm_core_utils.exception_mapping_utils", "exception_type"), + "get_error_message": ("litellm.litellm_core_utils.exception_mapping_utils", "get_error_message"), + "_get_response_headers": ("litellm.litellm_core_utils.exception_mapping_utils", "_get_response_headers"), + "get_llm_provider": ("litellm.litellm_core_utils.get_llm_provider_logic", "get_llm_provider"), + "_is_non_openai_azure_model": ( + "litellm.litellm_core_utils.get_llm_provider_logic", + "is_non_openai_azure_model", + ), + "is_non_openai_azure_model": ("litellm.litellm_core_utils.get_llm_provider_logic", "is_non_openai_azure_model"), + "get_supported_openai_params": ( + "litellm.litellm_core_utils.get_supported_openai_params", + "get_supported_openai_params", + ), + "LiteLLMResponseObjectHandler": ( + "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", + "LiteLLMResponseObjectHandler", + ), + "_handle_invalid_parallel_tool_calls": ( + "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", + "handle_invalid_parallel_tool_calls", + ), + "handle_invalid_parallel_tool_calls": ( + "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", + "handle_invalid_parallel_tool_calls", + ), + "convert_to_model_response_object": ( + "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", + "convert_to_model_response_object", + ), + "convert_to_streaming_response": ( + "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", + "convert_to_streaming_response", + ), + "convert_to_streaming_response_async": ( + "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", + "convert_to_streaming_response_async", + ), + "get_api_base": ("litellm.litellm_core_utils.llm_response_utils.get_api_base", "get_api_base"), + "ResponseMetadata": ("litellm.litellm_core_utils.llm_response_utils.response_metadata", "ResponseMetadata"), + "_parse_content_for_reasoning": ( + "litellm.litellm_core_utils.prompt_templates.common_utils", + "parse_content_for_reasoning", + ), + "parse_content_for_reasoning": ( + "litellm.litellm_core_utils.prompt_templates.common_utils", + "parse_content_for_reasoning", + ), + "LiteLLMLoggingObject": ("litellm.litellm_core_utils.redact_messages", "LiteLLMLoggingObject"), + "redact_message_input_output_from_logging": ( + "litellm.litellm_core_utils.redact_messages", + "redact_message_input_output_from_logging", + ), + "CustomStreamWrapper": ("litellm.litellm_core_utils.streaming_handler", "CustomStreamWrapper"), + "BaseGoogleGenAIGenerateContentConfig": ( + "litellm.llms.base_llm.google_genai.transformation", + "BaseGoogleGenAIGenerateContentConfig", + ), + "BaseSearchConfig": ("litellm.llms.base_llm.search.transformation", "BaseSearchConfig"), + "BaseTextToSpeechConfig": ("litellm.llms.base_llm.text_to_speech.transformation", "BaseTextToSpeechConfig"), + "BedrockModelInfo": ("litellm.llms.bedrock.common_utils", "BedrockModelInfo"), + "CohereModelInfo": ("litellm.llms.cohere.common_utils", "CohereModelInfo"), + "Rules": ("litellm.litellm_core_utils.rules", "Rules"), + "AsyncHTTPHandler": ("litellm.llms.custom_httpx.http_handler", "AsyncHTTPHandler"), + "HTTPHandler": ("litellm.llms.custom_httpx.http_handler", "HTTPHandler"), + "get_num_retries_from_retry_policy": ( + "litellm.litellm_core_utils.get_retry_from_policy", + "get_num_retries_from_retry_policy", + ), + "reset_retry_policy": ("litellm.litellm_core_utils.get_retry_from_policy", "reset_retry_policy"), + "get_coroutine_checker": ("litellm.litellm_core_utils.cached_imports", "get_coroutine_checker"), + "get_litellm_logging_class": ("litellm.litellm_core_utils.cached_imports", "get_litellm_logging_class"), + "get_set_callbacks": ("litellm.litellm_core_utils.cached_imports", "get_set_callbacks"), + "get_litellm_metadata_from_kwargs": ( + "litellm.litellm_core_utils.core_helpers", + "get_litellm_metadata_from_kwargs", + ), + "map_finish_reason": ("litellm.litellm_core_utils.core_helpers", "map_finish_reason"), + "process_response_headers": ("litellm.litellm_core_utils.core_helpers", "process_response_headers"), + "delete_nested_value": ("litellm.litellm_core_utils.dot_notation_indexing", "delete_nested_value"), + "is_nested_path": ("litellm.litellm_core_utils.dot_notation_indexing", "is_nested_path"), + "_get_base_model_from_litellm_call_metadata": ( + "litellm.litellm_core_utils.get_litellm_params", + "get_base_model_from_litellm_call_metadata", + ), + "get_base_model_from_litellm_call_metadata": ( + "litellm.litellm_core_utils.get_litellm_params", + "get_base_model_from_litellm_call_metadata", + ), + "get_litellm_params": ("litellm.litellm_core_utils.get_litellm_params", "get_litellm_params"), + "_ensure_extra_body_is_safe": ("litellm.litellm_core_utils.llm_request_utils", "ensure_extra_body_is_safe"), + "ensure_extra_body_is_safe": ("litellm.litellm_core_utils.llm_request_utils", "ensure_extra_body_is_safe"), + "get_formatted_prompt": ( + "litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt", + "get_formatted_prompt", + ), + "get_response_headers": ("litellm.litellm_core_utils.llm_response_utils.get_headers", "get_response_headers"), + "update_response_metadata": ( + "litellm.litellm_core_utils.llm_response_utils.response_metadata", + "update_response_metadata", + ), + "executor": ("litellm.litellm_core_utils.thread_pool_executor", "executor"), + "BaseAnthropicMessagesConfig": ( + "litellm.llms.base_llm.anthropic_messages.transformation", + "BaseAnthropicMessagesConfig", + ), + "BaseAudioTranscriptionConfig": ( + "litellm.llms.base_llm.audio_transcription.transformation", + "BaseAudioTranscriptionConfig", + ), + "BaseBatchesConfig": ("litellm.llms.base_llm.batches.transformation", "BaseBatchesConfig"), + "BaseContainerConfig": ("litellm.llms.base_llm.containers.transformation", "BaseContainerConfig"), + "BaseEmbeddingConfig": ("litellm.llms.base_llm.embedding.transformation", "BaseEmbeddingConfig"), + "BaseImageEditConfig": ("litellm.llms.base_llm.image_edit.transformation", "BaseImageEditConfig"), + "BaseImageGenerationConfig": ( + "litellm.llms.base_llm.image_generation.transformation", + "BaseImageGenerationConfig", + ), + "BaseImageVariationConfig": ( + "litellm.llms.base_llm.image_variations.transformation", + "BaseImageVariationConfig", + ), + "BasePassthroughConfig": ("litellm.llms.base_llm.passthrough.transformation", "BasePassthroughConfig"), + "BaseRealtimeConfig": ("litellm.llms.base_llm.realtime.transformation", "BaseRealtimeConfig"), + "BaseRerankConfig": ("litellm.llms.base_llm.rerank.transformation", "BaseRerankConfig"), + "BaseVectorStoreConfig": ("litellm.llms.base_llm.vector_store.transformation", "BaseVectorStoreConfig"), + "BaseVectorStoreFilesConfig": ( + "litellm.llms.base_llm.vector_store_files.transformation", + "BaseVectorStoreFilesConfig", + ), + "BaseVideoConfig": ("litellm.llms.base_llm.videos.transformation", "BaseVideoConfig"), + "ANTHROPIC_API_ONLY_HEADERS": ("litellm.types.llms.anthropic", "ANTHROPIC_API_ONLY_HEADERS"), + "AnthropicThinkingParam": ("litellm.types.llms.anthropic", "AnthropicThinkingParam"), + "RerankResponse": ("litellm.types.rerank", "RerankResponse"), + "ChatCompletionDeltaToolCallChunk": ("litellm.types.llms.openai", "ChatCompletionDeltaToolCallChunk"), + "ChatCompletionToolCallChunk": ("litellm.types.llms.openai", "ChatCompletionToolCallChunk"), + "ChatCompletionToolCallFunctionChunk": ("litellm.types.llms.openai", "ChatCompletionToolCallFunctionChunk"), + "LiteLLM_Params": ("litellm.types.router", "LiteLLM_Params"), + } ) - -# Utils names that support lazy loading via _lazy_import_utils -UTILS_NAMES: Final = ( - "run_server", - "exception_type", - "get_optional_params", - "get_response_string", - "token_counter", - "create_pretrained_tokenizer", - "create_tokenizer", - "supports_function_calling", - "supports_web_search", - "supports_tool_search", - "supports_url_context", - "supports_continuation", - "supports_response_schema", - "supports_parallel_function_calling", - "supports_vision", - "supports_audio_input", - "supports_video_input", - "supports_box_questions", - "supports_audio_output", - "supports_system_messages", - "supports_reasoning", - "get_litellm_params", - "acreate", - "get_max_tokens", - "get_model_info", - "register_prompt_template", - "validate_environment", - "check_valid_key", - "register_model", - "encode", - "decode", - "calculate_retry_after", - "_calculate_retry_after", - "should_retry", - "_should_retry", - "get_supported_openai_params", - "get_api_base", - "get_first_chars_messages", - "ModelResponse", - "ModelResponseStream", - "EmbeddingResponse", - "ImageResponse", - "TranscriptionResponse", - "TextCompletionResponse", - "get_provider_fields", - "ModelResponseListIterator", - "get_valid_models", - "timeout", - "get_llm_provider", - "remove_index_from_tool_calls", -) - -# Token counter names that support lazy loading via _lazy_import_token_counter -TOKEN_COUNTER_NAMES: Final = ("get_modified_max_tokens",) - -# LLM client cache names that support lazy loading via _lazy_import_llm_client_cache -LLM_CLIENT_CACHE_NAMES: Final = ( - "LLMClientCache", - "in_memory_llm_clients_cache", -) - -# Bedrock type names that support lazy loading via _lazy_import_bedrock_types -BEDROCK_TYPES_NAMES: Final = ("COHERE_EMBEDDING_INPUT_TYPES",) - -# Common types from litellm.types.utils that support lazy loading via -# _lazy_import_types_utils -TYPES_UTILS_NAMES: Final = ( - "ImageObject", - "BudgetConfig", - "all_litellm_params", - "_litellm_completion_params", - "CredentialItem", - "PriorityReservationDict", - "StandardKeyGenerationConfig", - "SearchProviders", - "GenericStreamingChunk", -) - -# Caching / cache classes that support lazy loading via _lazy_import_caching -CACHING_NAMES: Final = ( - "Cache", - "DualCache", - "RedisCache", - "InMemoryCache", -) - -# HTTP handler names that support lazy loading via _lazy_import_http_handlers -HTTP_HANDLER_NAMES: Final = ( - "module_level_aclient", - "module_level_client", -) - -# Dotprompt integration names that support lazy loading via _lazy_import_dotprompt -DOTPROMPT_NAMES: Final = ( - "global_prompt_manager", - "global_prompt_directory", - "set_global_prompt_directory", -) - -# LLM config classes that support lazy loading via _lazy_import_llm_configs -LLM_CONFIG_NAMES: Final = ( - "AmazonConverseConfig", - "OpenAILikeChatConfig", - "GaladrielChatConfig", - "GithubChatConfig", - "AzureAnthropicConfig", - "BytezChatConfig", - "CompactifAIChatConfig", - "EmpowerChatConfig", - "MinimaxChatConfig", - "AiohttpOpenAIChatConfig", - "HuggingFaceChatConfig", - "HuggingFaceEmbeddingConfig", - "OobaboogaConfig", - "MaritalkConfig", - "OpenrouterConfig", - "DataRobotConfig", - "AnthropicConfig", - "BedrockClaudePlatformConfig", - "AnthropicTextConfig", - "GroqSTTConfig", - "TritonConfig", - "TritonGenerateConfig", - "TritonInferConfig", - "TritonEmbeddingConfig", - "HuggingFaceRerankConfig", - "DatabricksConfig", - "DatabricksEmbeddingConfig", - "PredibaseConfig", - "ReplicateConfig", - "SnowflakeConfig", - "CohereRerankConfig", - "CohereRerankV2Config", - "AzureAIRerankConfig", - "InfinityRerankConfig", - "JinaAIRerankConfig", - "ScalewayRerankConfig", - "DeepinfraRerankConfig", - "HostedVLLMRerankConfig", - "PerplexityDecisionsConfig", - "TypeSafeDecisionsConfig", - "ParallelAIDecisionsConfig", - "NaceDecisionsConfig", - "OpenRouterDecisionsConfig", - "FireworksAIDecisionsConfig", - "CloudflareDecisionsConfig", - "CloudflareExtractionConfig", - "StrandsDeciderDecisionsConfig", - "DatabricksDecisionsConfig", - "HostedVLLMDecisionsConfig", - "LMStudioDecisionsConfig", - "OllamaDecisionsConfig", - "LayaDecisionsConfig", - "BespokeDecisionsConfig", - "EmpirioLabsDecisionsConfig", - "EuJevDecisionsConfig", - "OpenAIDecisionsConfig", - "AzureDecisionsConfig", - "AzureAIDecisionsConfig", - "NvidiaNimRerankConfig", - "NvidiaNimRankingConfig", - "VertexAIRerankConfig", - "FireworksAIRerankConfig", - "VoyageRerankConfig", - "IBMWatsonXRerankConfig", - "ClarifaiConfig", - "AI21ChatConfig", - "LlamaAPIConfig", - "TogetherAITextCompletionConfig", - "CloudflareChatConfig", - "NovitaConfig", - "PetalsConfig", - "OllamaChatConfig", - "OllamaConfig", - "SagemakerConfig", - "SagemakerChatConfig", - "SagemakerNovaConfig", - "CohereChatConfig", - "AnthropicMessagesConfig", - "BedrockClaudePlatformMessagesConfig", - "AmazonAnthropicClaudeMessagesConfig", - "AmazonMantleMessagesConfig", - "BedrockMantleAnthropicMessagesConfig", - "TogetherAIConfig", - "TogetherAIChatConfig", - "NLPCloudConfig", - "VertexGeminiConfig", - "GoogleAIStudioGeminiConfig", - "VertexAIAnthropicConfig", - "VertexAILlama3Config", - "VertexAIAi21Config", - "VertexAIMistralConfig", - "AmazonCohereChatConfig", - "AmazonBedrockGlobalConfig", - "AmazonAI21Config", - "AmazonInvokeNovaConfig", - "AmazonQwen2Config", - "AmazonQwen3Config", - # Aliases for backwards compatibility - "VertexAIConfig", # Alias for VertexGeminiConfig - "GeminiConfig", # Alias for GoogleAIStudioGeminiConfig - "AmazonAnthropicConfig", - "AmazonAnthropicClaudeConfig", - "AmazonCohereConfig", - "AmazonLlamaConfig", - "AmazonDeepSeekR1Config", - "AmazonMistralConfig", - "AmazonMoonshotConfig", - "AmazonTitanConfig", - "AmazonTwelveLabsPegasusConfig", - "AmazonInvokeConfig", - "AmazonBedrockOpenAIConfig", - "AmazonBedrockRuntimeChatCompletionsConfig", - "AmazonStabilityConfig", - "AmazonStability3Config", - "AmazonNovaCanvasConfig", - "AmazonTitanG1Config", - "AmazonTitanMultimodalEmbeddingG1Config", - "CohereV2ChatConfig", - "BedrockCohereEmbeddingConfig", - "TwelveLabsMarengoEmbeddingConfig", - "AmazonNovaEmbeddingConfig", - "OpenAIConfig", - "MistralEmbeddingConfig", - "OpenAIImageVariationConfig", - "DeepInfraConfig", - "ClfAiGatewayConfig", - "DeepgramAudioTranscriptionConfig", - "TopazImageVariationConfig", - "OpenAITextCompletionConfig", - "GroqChatConfig", - "BedrockMantleChatConfig", - "A2AConfig", - "GenAIHubOrchestrationConfig", - "VoyageEmbeddingConfig", - "VoyageContextualEmbeddingConfig", - "VoyageMultimodalEmbeddingConfig", - "InfinityEmbeddingConfig", - "PerplexityEmbeddingConfig", - "AzureAIStudioConfig", - "MistralConfig", - "OpenAIResponsesAPIConfig", - "AzureOpenAIResponsesAPIConfig", - "AzureOpenAIOSeriesResponsesAPIConfig", - "AzureAIResponsesAPIConfig", - "XAIResponsesAPIConfig", - "LiteLLMProxyResponsesAPIConfig", - "HostedVLLMResponsesAPIConfig", - "FireworksAIResponsesAPIConfig", - "VolcEngineResponsesAPIConfig", - "PerplexityResponsesConfig", - "DatabricksResponsesAPIConfig", - "OpenRouterResponsesAPIConfig", - "BedrockOpenAIResponsesConfig", - "BedrockMantleResponsesAPIConfig", - "GoogleAIStudioInteractionsConfig", - "VertexAIInteractionsConfig", - "OpenAIOSeriesConfig", - "AnthropicSkillsConfig", - "BaseSkillsAPIConfig", - "GradientAIConfig", - # Alias for backwards compatibility - "OpenAIO1Config", # Alias for OpenAIOSeriesConfig - "OpenAIGPTConfig", - "OpenAIGPT5Config", - "OpenAIWhisperAudioTranscriptionConfig", - "OpenAIGPTAudioTranscriptionConfig", - "OpenAIGPTAudioConfig", - "NvidiaNimConfig", - "NvidiaNimEmbeddingConfig", - "FeatherlessAIConfig", - "ScaleDownChatConfig", - "CerebrasConfig", - "NadirConfig", - "BasetenConfig", - "SambanovaConfig", - "SambaNovaEmbeddingConfig", - "FireworksAIConfig", - "FireworksAITextCompletionConfig", - "FireworksAIEmbeddingConfig", - "FriendliaiChatConfig", - "JinaAIEmbeddingConfig", - "XAIChatConfig", - "ZAIChatConfig", - "AIMLChatConfig", - "VolcEngineChatConfig", - "CodestralTextCompletionConfig", - "InceptionTextCompletionConfig", - "AzureOpenAIAssistantsAPIConfig", - "HerokuChatConfig", - "CometAPIConfig", - "AzureOpenAIConfig", - "AzureOpenAIGPT5Config", - "AzureOpenAITextConfig", - "AzureSpeechAudioTranscriptionConfig", - "HostedVLLMChatConfig", - "HostedVLLMEmbeddingConfig", - # Alias for backwards compatibility - "VolcEngineConfig", # Alias for VolcEngineChatConfig - "LlamafileChatConfig", - "LiteLLMProxyChatConfig", - "VLLMConfig", - "DeepSeekChatConfig", - "TencentChatConfig", - "LMStudioChatConfig", - "LmStudioEmbeddingConfig", - "NscaleConfig", - "PerplexityChatConfig", - "AzureOpenAIO1Config", - "IBMWatsonXAIConfig", - "IBMWatsonXChatConfig", - "IBMWatsonXEmbeddingConfig", - "GenAIHubEmbeddingConfig", - "IBMWatsonXAudioTranscriptionConfig", - "GithubCopilotConfig", - "GithubCopilotResponsesAPIConfig", - "ChatGPTConfig", - "ChatGPTResponsesAPIConfig", - "ManusResponsesAPIConfig", - "GithubCopilotEmbeddingConfig", - "NebiusConfig", - "WandbConfig", - "GigaChatConfig", - "GigaChatEmbeddingConfig", - "DashScopeChatConfig", - "QwenCloudChatConfig", - "QwenAIPlatformChatConfig", - "ModelScopeChatConfig", - "MoonshotChatConfig", - "DockerModelRunnerChatConfig", - "V0ChatConfig", - "OCIChatConfig", - "MorphChatConfig", - "RAGFlowConfig", - "LambdaAIChatConfig", - "InceptionChatConfig", - "HyperbolicChatConfig", - "VercelAIGatewayConfig", - "EdenAIChatConfig", - "EdenAIResponsesAPIConfig", - "EdenAIAnthropicMessagesConfig", - "EdenAIEmbeddingConfig", - "EdenAIAudioTranscriptionConfig", - "EdenAITextToSpeechConfig", - "EdenAIImageGenerationConfig", - "EdenAIVideoConfig", - "FalAIChatConfig", - "FalAIError", - "OVHCloudChatConfig", - "OVHCloudEmbeddingConfig", - "CometAPIEmbeddingConfig", - "LemonadeChatConfig", - "SnowflakeEmbeddingConfig", - "AmazonNovaChatConfig", - "SonioxAudioTranscriptionConfig", - "GDCGeminiConfig", -) - -# Types that support lazy loading via _lazy_import_types -TYPES_NAMES: Final = ( - "GuardrailItem", - "DefaultTeamSSOParams", - "LiteLLM_UpperboundKeyGenerateParams", - "KeyManagementSystem", - "PriorityReservationSettings", - "CustomLogger", - "LoggingCallbackManager", - "DatadogLLMObsInitParams", - # Note: LlmProviders is NOT lazy-loaded because it's imported during import time - # in multiple places including openai.py (via main import) - # Note: KeyManagementSettings is NOT lazy-loaded because _key_management_settings - # is accessed during import time in secret_managers/main.py -) - -# LLM provider logic names that support lazy loading via _lazy_import_llm_provider_logic -LLM_PROVIDER_LOGIC_NAMES: Final = ( - "get_llm_provider", - "remove_index_from_tool_calls", -) - -# Utils module names that support lazy loading via _lazy_import_utils_module -# These are attributes accessed from litellm.utils module -UTILS_MODULE_NAMES: Final = ( - "encoding", - "BaseVectorStore", - "CredentialAccessor", - "exception_type", - "get_error_message", - "_get_response_headers", - "get_llm_provider", - "_is_non_openai_azure_model", - "is_non_openai_azure_model", - "get_supported_openai_params", - "LiteLLMResponseObjectHandler", - "_handle_invalid_parallel_tool_calls", - "handle_invalid_parallel_tool_calls", - "convert_to_model_response_object", - "convert_to_streaming_response", - "convert_to_streaming_response_async", - "get_api_base", - "ResponseMetadata", - "_parse_content_for_reasoning", - "parse_content_for_reasoning", - "LiteLLMLoggingObject", - "redact_message_input_output_from_logging", - "CustomStreamWrapper", - "BaseGoogleGenAIGenerateContentConfig", - "BaseSearchConfig", - "BaseTextToSpeechConfig", - "BedrockModelInfo", - "CohereModelInfo", - "Rules", - "AsyncHTTPHandler", - "HTTPHandler", - "get_num_retries_from_retry_policy", - "reset_retry_policy", - "get_coroutine_checker", - "get_litellm_logging_class", - "get_set_callbacks", - "get_litellm_metadata_from_kwargs", - "map_finish_reason", - "process_response_headers", - "delete_nested_value", - "is_nested_path", - "_get_base_model_from_litellm_call_metadata", - "get_base_model_from_litellm_call_metadata", - "get_litellm_params", - "_ensure_extra_body_is_safe", - "ensure_extra_body_is_safe", - "get_formatted_prompt", - "get_response_headers", - "update_response_metadata", - "executor", - "BaseAnthropicMessagesConfig", - "BaseAudioTranscriptionConfig", - "BaseBatchesConfig", - "BaseContainerConfig", - "BaseEmbeddingConfig", - "BaseImageEditConfig", - "BaseImageGenerationConfig", - "BaseImageVariationConfig", - "BasePassthroughConfig", - "BaseRealtimeConfig", - "BaseRerankConfig", - "BaseVectorStoreConfig", - "BaseVectorStoreFilesConfig", - "BaseVideoConfig", - "ANTHROPIC_API_ONLY_HEADERS", - "AnthropicThinkingParam", - "RerankResponse", - "ChatCompletionDeltaToolCallChunk", - "ChatCompletionToolCallChunk", - "ChatCompletionToolCallFunctionChunk", - "LiteLLM_Params", -) - -# Import maps for registry pattern - reduces repetition -_UTILS_IMPORT_MAP: Final = { - "run_server": ("litellm.proxy.proxy_cli", "run_server"), - "exception_type": (".utils", "exception_type"), - "get_optional_params": (".utils", "get_optional_params"), - "get_response_string": (".utils", "get_response_string"), - "token_counter": (".utils", "token_counter"), - "create_pretrained_tokenizer": (".utils", "create_pretrained_tokenizer"), - "create_tokenizer": (".utils", "create_tokenizer"), - "supports_function_calling": (".utils", "supports_function_calling"), - "supports_web_search": (".utils", "supports_web_search"), - "supports_tool_search": (".utils", "supports_tool_search"), - "supports_url_context": (".utils", "supports_url_context"), - "supports_continuation": (".utils", "supports_continuation"), - "supports_response_schema": (".utils", "supports_response_schema"), - "supports_parallel_function_calling": ( - ".utils", - "supports_parallel_function_calling", - ), - "supports_vision": (".utils", "supports_vision"), - "supports_audio_input": (".utils", "supports_audio_input"), - "supports_video_input": (".utils", "supports_video_input"), - "supports_box_questions": (".utils", "supports_box_questions"), - "supports_audio_output": (".utils", "supports_audio_output"), - "supports_system_messages": (".utils", "supports_system_messages"), - "supports_reasoning": (".utils", "supports_reasoning"), - "get_litellm_params": (".utils", "get_litellm_params"), - "acreate": (".utils", "acreate"), - "get_max_tokens": (".utils", "get_max_tokens"), - "get_model_info": (".utils", "get_model_info"), - "register_prompt_template": (".utils", "register_prompt_template"), - "validate_environment": (".utils", "validate_environment"), - "check_valid_key": (".utils", "check_valid_key"), - "register_model": (".utils", "register_model"), - "encode": (".utils", "encode"), - "decode": (".utils", "decode"), - "calculate_retry_after": (".utils", "calculate_retry_after"), - "_calculate_retry_after": (".utils", "calculate_retry_after"), - "should_retry": (".utils", "should_retry"), - "_should_retry": (".utils", "should_retry"), - "get_supported_openai_params": (".utils", "get_supported_openai_params"), - "get_api_base": (".utils", "get_api_base"), - "get_first_chars_messages": (".utils", "get_first_chars_messages"), - "ModelResponse": (".utils", "ModelResponse"), - "ModelResponseStream": (".utils", "ModelResponseStream"), - "EmbeddingResponse": (".utils", "EmbeddingResponse"), - "ImageResponse": (".utils", "ImageResponse"), - "TranscriptionResponse": (".utils", "TranscriptionResponse"), - "TextCompletionResponse": (".utils", "TextCompletionResponse"), - "get_provider_fields": (".utils", "get_provider_fields"), - "ModelResponseListIterator": (".utils", "ModelResponseListIterator"), - "get_valid_models": (".utils", "get_valid_models"), - "timeout": (".timeout", "timeout"), - "get_llm_provider": ( - "litellm.litellm_core_utils.get_llm_provider_logic", - "get_llm_provider", - ), - "remove_index_from_tool_calls": ( - "litellm.litellm_core_utils.core_helpers", - "remove_index_from_tool_calls", - ), -} - -_COST_CALCULATOR_IMPORT_MAP: Final = { - "completion_cost": (".cost_calculator", "completion_cost"), - "cost_per_token": (".cost_calculator", "cost_per_token"), - "response_cost_calculator": (".cost_calculator", "response_cost_calculator"), -} - -_TYPES_UTILS_IMPORT_MAP: Final = { - "ImageObject": (".types.utils", "ImageObject"), - "BudgetConfig": (".types.utils", "BudgetConfig"), - "all_litellm_params": (".types.utils", "all_litellm_params"), - "_litellm_completion_params": (".types.utils", "all_litellm_params"), # Alias - "CredentialItem": (".types.utils", "CredentialItem"), - "PriorityReservationDict": (".types.utils", "PriorityReservationDict"), - "StandardKeyGenerationConfig": (".types.utils", "StandardKeyGenerationConfig"), - "SearchProviders": (".types.utils", "SearchProviders"), - "GenericStreamingChunk": (".types.utils", "GenericStreamingChunk"), -} - -_TOKEN_COUNTER_IMPORT_MAP: Final = { - "get_modified_max_tokens": ( - "litellm.litellm_core_utils.token_counter", - "get_modified_max_tokens", - ), -} - -_BEDROCK_TYPES_IMPORT_MAP: Final = { - "COHERE_EMBEDDING_INPUT_TYPES": ( - "litellm.types.llms.bedrock", - "COHERE_EMBEDDING_INPUT_TYPES", - ), -} - -_CACHING_IMPORT_MAP: Final = { - "Cache": ("litellm.caching.caching", "Cache"), - "DualCache": ("litellm.caching.caching", "DualCache"), - "RedisCache": ("litellm.caching.caching", "RedisCache"), - "InMemoryCache": ("litellm.caching.caching", "InMemoryCache"), -} - -_LITELLM_LOGGING_IMPORT_MAP: Final = { - "Logging": ("litellm.litellm_core_utils.litellm_logging", "Logging"), - "modify_integration": ( - "litellm.litellm_core_utils.litellm_logging", - "modify_integration", - ), -} - -_DOTPROMPT_IMPORT_MAP: Final = { - "global_prompt_manager": ( - "litellm.integrations.dotprompt", - "global_prompt_manager", - ), - "global_prompt_directory": ( - "litellm.integrations.dotprompt", - "global_prompt_directory", - ), - "set_global_prompt_directory": ( - "litellm.integrations.dotprompt", - "set_global_prompt_directory", - ), -} - -_TYPES_IMPORT_MAP: Final = { - "GuardrailItem": ("litellm.types.guardrails", "GuardrailItem"), - "DefaultTeamSSOParams": ( - "litellm.types.proxy.management_endpoints.ui_sso", - "DefaultTeamSSOParams", - ), - "LiteLLM_UpperboundKeyGenerateParams": ( - "litellm.types.proxy.management_endpoints.ui_sso", - "LiteLLM_UpperboundKeyGenerateParams", - ), - "KeyManagementSystem": ( - "litellm.types.secret_managers.main", - "KeyManagementSystem", - ), - "PriorityReservationSettings": ( - "litellm.types.utils", - "PriorityReservationSettings", - ), - "CustomLogger": ("litellm.integrations.custom_logger", "CustomLogger"), - "LoggingCallbackManager": ( - "litellm.litellm_core_utils.logging_callback_manager", - "LoggingCallbackManager", - ), - "DatadogLLMObsInitParams": ( - "litellm.types.integrations.datadog_llm_obs", - "DatadogLLMObsInitParams", - ), -} - -_LLM_PROVIDER_LOGIC_IMPORT_MAP: Final = { - "get_llm_provider": ( - "litellm.litellm_core_utils.get_llm_provider_logic", - "get_llm_provider", - ), - "remove_index_from_tool_calls": ( - "litellm.litellm_core_utils.core_helpers", - "remove_index_from_tool_calls", - ), -} - -_LLM_CONFIGS_IMPORT_MAP: Final = { - "AmazonConverseConfig": ( - ".llms.bedrock.chat.converse_transformation", - "AmazonConverseConfig", - ), - "OpenAILikeChatConfig": (".llms.openai_like.chat.handler", "OpenAILikeChatConfig"), - "GaladrielChatConfig": ( - ".llms.galadriel.chat.transformation", - "GaladrielChatConfig", - ), - "GithubChatConfig": (".llms.github.chat.transformation", "GithubChatConfig"), - "AzureAnthropicConfig": ( - ".llms.azure_ai.anthropic.transformation", - "AzureAnthropicConfig", - ), - "BytezChatConfig": (".llms.bytez.chat.transformation", "BytezChatConfig"), - "CompactifAIChatConfig": ( - ".llms.compactifai.chat.transformation", - "CompactifAIChatConfig", - ), - "EmpowerChatConfig": (".llms.empower.chat.transformation", "EmpowerChatConfig"), - "MinimaxChatConfig": (".llms.minimax.chat.transformation", "MinimaxChatConfig"), - "AiohttpOpenAIChatConfig": ( - ".llms.aiohttp_openai.chat.transformation", - "AiohttpOpenAIChatConfig", - ), - "HuggingFaceChatConfig": ( - ".llms.huggingface.chat.transformation", - "HuggingFaceChatConfig", - ), - "HuggingFaceEmbeddingConfig": ( - ".llms.huggingface.embedding.transformation", - "HuggingFaceEmbeddingConfig", - ), - "OobaboogaConfig": (".llms.oobabooga.chat.transformation", "OobaboogaConfig"), - "MaritalkConfig": (".llms.maritalk", "MaritalkConfig"), - "OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"), - "DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"), - "AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"), - "BedrockClaudePlatformConfig": ( - ".llms.bedrock.claude_platform.transformation", - "BedrockClaudePlatformConfig", - ), - "AnthropicTextConfig": ( - ".llms.anthropic.completion.transformation", - "AnthropicTextConfig", - ), - "GroqSTTConfig": (".llms.groq.stt.transformation", "GroqSTTConfig"), - "TritonConfig": (".llms.triton.completion.transformation", "TritonConfig"), - "TritonGenerateConfig": ( - ".llms.triton.completion.transformation", - "TritonGenerateConfig", - ), - "TritonInferConfig": ( - ".llms.triton.completion.transformation", - "TritonInferConfig", - ), - "TritonEmbeddingConfig": ( - ".llms.triton.embedding.transformation", - "TritonEmbeddingConfig", - ), - "HuggingFaceRerankConfig": ( - ".llms.huggingface.rerank.transformation", - "HuggingFaceRerankConfig", - ), - "DatabricksConfig": (".llms.databricks.chat.transformation", "DatabricksConfig"), - "DatabricksEmbeddingConfig": ( - ".llms.databricks.embed.transformation", - "DatabricksEmbeddingConfig", - ), - "PredibaseConfig": (".llms.predibase.chat.transformation", "PredibaseConfig"), - "ReplicateConfig": (".llms.replicate.chat.transformation", "ReplicateConfig"), - "SnowflakeConfig": (".llms.snowflake.chat.transformation", "SnowflakeConfig"), - "CohereRerankConfig": (".llms.cohere.rerank.transformation", "CohereRerankConfig"), - "CohereRerankV2Config": ( - ".llms.cohere.rerank_v2.transformation", - "CohereRerankV2Config", - ), - "AzureAIRerankConfig": ( - ".llms.azure_ai.rerank.transformation", - "AzureAIRerankConfig", - ), - "InfinityRerankConfig": ( - ".llms.infinity.rerank.transformation", - "InfinityRerankConfig", - ), - "JinaAIRerankConfig": (".llms.jina_ai.rerank.transformation", "JinaAIRerankConfig"), - "ScalewayRerankConfig": (".llms.scaleway.rerank.transformation", "ScalewayRerankConfig"), - "DeepinfraRerankConfig": ( - ".llms.deepinfra.rerank.transformation", - "DeepinfraRerankConfig", - ), - "HostedVLLMRerankConfig": ( - ".llms.hosted_vllm.rerank.transformation", - "HostedVLLMRerankConfig", - ), - "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), - "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"), - "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), - "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"), - "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), - "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"), - "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), - "CloudflareExtractionConfig": (".llms.cloudflare.extraction.transformation", "CloudflareExtractionConfig"), - "StrandsDeciderDecisionsConfig": ( - ".llms.strands_decider.decisions.transformation", - "StrandsDeciderDecisionsConfig", - ), - "DatabricksDecisionsConfig": (".llms.databricks.decisions.transformation", "DatabricksDecisionsConfig"), - "HostedVLLMDecisionsConfig": ( - ".llms.hosted_vllm.decisions.transformation", - "HostedVLLMDecisionsConfig", - ), - "LMStudioDecisionsConfig": (".llms.lm_studio.decisions.transformation", "LMStudioDecisionsConfig"), - "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"), - "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), - "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), - "EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"), - "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"), - "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), - "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), - "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), - "NvidiaNimRerankConfig": ( - ".llms.nvidia_nim.rerank.transformation", - "NvidiaNimRerankConfig", - ), - "NvidiaNimRankingConfig": ( - ".llms.nvidia_nim.rerank.ranking_transformation", - "NvidiaNimRankingConfig", - ), - "VertexAIRerankConfig": ( - ".llms.vertex_ai.rerank.transformation", - "VertexAIRerankConfig", - ), - "FireworksAIRerankConfig": ( - ".llms.fireworks_ai.rerank.transformation", - "FireworksAIRerankConfig", - ), - "VoyageRerankConfig": (".llms.voyage.rerank.transformation", "VoyageRerankConfig"), - "IBMWatsonXRerankConfig": ( - ".llms.watsonx.rerank.transformation", - "IBMWatsonXRerankConfig", - ), - "ClarifaiConfig": (".llms.clarifai.chat.transformation", "ClarifaiConfig"), - "AI21ChatConfig": (".llms.ai21.chat.transformation", "AI21ChatConfig"), - "LlamaAPIConfig": (".llms.meta_llama.chat.transformation", "LlamaAPIConfig"), - "TogetherAITextCompletionConfig": ( - ".llms.together_ai.completion.transformation", - "TogetherAITextCompletionConfig", - ), - "CloudflareChatConfig": ( - ".llms.cloudflare.chat.transformation", - "CloudflareChatConfig", - ), - "NovitaConfig": (".llms.novita.chat.transformation", "NovitaConfig"), - "PetalsConfig": (".llms.petals.completion.transformation", "PetalsConfig"), - "OllamaChatConfig": (".llms.ollama.chat.transformation", "OllamaChatConfig"), - "OllamaConfig": (".llms.ollama.completion.transformation", "OllamaConfig"), - "SagemakerConfig": (".llms.sagemaker.completion.transformation", "SagemakerConfig"), - "SagemakerChatConfig": ( - ".llms.sagemaker.chat.transformation", - "SagemakerChatConfig", - ), - "SagemakerNovaConfig": ( - ".llms.sagemaker.nova.transformation", - "SagemakerNovaConfig", - ), - "CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"), - "AnthropicMessagesConfig": ( - ".llms.anthropic.messages.native.transformation", - "AnthropicMessagesConfig", - ), - "BedrockClaudePlatformMessagesConfig": ( - ".llms.bedrock.claude_platform.messages_transformation", - "BedrockClaudePlatformMessagesConfig", - ), - "AmazonAnthropicClaudeMessagesConfig": ( - ".llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation", - "AmazonAnthropicClaudeMessagesConfig", - ), - "AmazonMantleMessagesConfig": ( - ".llms.bedrock.messages.mantle_transformation", - "AmazonMantleMessagesConfig", - ), - "BedrockMantleAnthropicMessagesConfig": ( - ".llms.bedrock_mantle.messages.transformation", - "BedrockMantleAnthropicMessagesConfig", - ), - "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"), - "TogetherAIChatConfig": ( - ".llms.together_ai.chat.transformation", - "TogetherAIChatConfig", - ), - "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"), - "VertexGeminiConfig": ( - ".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", - "VertexGeminiConfig", - ), - "GoogleAIStudioGeminiConfig": ( - ".llms.gemini.chat.transformation", - "GoogleAIStudioGeminiConfig", - ), - "VertexAIAnthropicConfig": ( - ".llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation", - "VertexAIAnthropicConfig", - ), - "VertexAILlama3Config": ( - ".llms.vertex_ai.vertex_ai_partner_models.llama.transformation", - "VertexAILlama3Config", - ), - "VertexAIAi21Config": ( - ".llms.vertex_ai.vertex_ai_partner_models.ai21.transformation", - "VertexAIAi21Config", - ), - "VertexAIMistralConfig": ( - ".llms.vertex_ai.vertex_ai_partner_models.mistral.transformation", - "VertexAIMistralConfig", - ), - "AmazonCohereChatConfig": ( - ".llms.bedrock.chat.invoke_handler", - "AmazonCohereChatConfig", - ), - "AmazonBedrockGlobalConfig": ( - ".llms.bedrock.common_utils", - "AmazonBedrockGlobalConfig", - ), - "AmazonAI21Config": ( - ".llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation", - "AmazonAI21Config", - ), - "AmazonInvokeNovaConfig": ( - ".llms.bedrock.chat.invoke_transformations.amazon_nova_transformation", - "AmazonInvokeNovaConfig", - ), - "AmazonQwen2Config": ( - ".llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation", - "AmazonQwen2Config", - ), - "AmazonQwen3Config": ( - ".llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation", - "AmazonQwen3Config", - ), - # Aliases for backwards compatibility - "VertexAIConfig": ( - ".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", - "VertexGeminiConfig", - ), # Alias - "GeminiConfig": ( - ".llms.gemini.chat.transformation", - "GoogleAIStudioGeminiConfig", - ), # Alias - "AmazonAnthropicConfig": ( - ".llms.bedrock.chat.invoke_transformations.anthropic_legacy_text_completions_transformation", - "AmazonAnthropicConfig", - ), - "AmazonAnthropicClaudeConfig": ( - ".llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation", - "AmazonAnthropicClaudeConfig", - ), - "AmazonCohereConfig": ( - ".llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation", - "AmazonCohereConfig", - ), - "AmazonLlamaConfig": ( - ".llms.bedrock.chat.invoke_transformations.amazon_llama_transformation", - "AmazonLlamaConfig", - ), - "AmazonDeepSeekR1Config": ( - ".llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation", - "AmazonDeepSeekR1Config", - ), - "AmazonMistralConfig": ( - ".llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation", - "AmazonMistralConfig", - ), - "AmazonMoonshotConfig": ( - ".llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation", - "AmazonMoonshotConfig", - ), - "AmazonTitanConfig": ( - ".llms.bedrock.chat.invoke_transformations.amazon_titan_transformation", - "AmazonTitanConfig", - ), - "AmazonTwelveLabsPegasusConfig": ( - ".llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation", - "AmazonTwelveLabsPegasusConfig", - ), - "AmazonInvokeConfig": ( - ".llms.bedrock.chat.invoke_transformations.base_invoke_transformation", - "AmazonInvokeConfig", - ), - "AmazonBedrockOpenAIConfig": ( - ".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation", - "AmazonBedrockOpenAIConfig", - ), - "AmazonBedrockRuntimeChatCompletionsConfig": ( - ".llms.bedrock.chat.chat_completions.transformation", - "AmazonBedrockRuntimeChatCompletionsConfig", - ), - "AmazonStabilityConfig": ( - ".llms.bedrock.image_generation.amazon_stability1_transformation", - "AmazonStabilityConfig", - ), - "AmazonStability3Config": ( - ".llms.bedrock.image_generation.amazon_stability3_transformation", - "AmazonStability3Config", - ), - "AmazonNovaCanvasConfig": ( - ".llms.bedrock.image_generation.amazon_nova_canvas_transformation", - "AmazonNovaCanvasConfig", - ), - "AmazonTitanG1Config": ( - ".llms.bedrock.embed.amazon_titan_g1_transformation", - "AmazonTitanG1Config", - ), - "AmazonTitanMultimodalEmbeddingG1Config": ( - ".llms.bedrock.embed.amazon_titan_multimodal_transformation", - "AmazonTitanMultimodalEmbeddingG1Config", - ), - "CohereV2ChatConfig": (".llms.cohere.chat.v2_transformation", "CohereV2ChatConfig"), - "BedrockCohereEmbeddingConfig": ( - ".llms.bedrock.embed.cohere_transformation", - "BedrockCohereEmbeddingConfig", - ), - "TwelveLabsMarengoEmbeddingConfig": ( - ".llms.bedrock.embed.twelvelabs_marengo_transformation", - "TwelveLabsMarengoEmbeddingConfig", - ), - "AmazonNovaEmbeddingConfig": ( - ".llms.bedrock.embed.amazon_nova_transformation", - "AmazonNovaEmbeddingConfig", - ), - "OpenAIConfig": (".llms.openai.openai", "OpenAIConfig"), - "MistralEmbeddingConfig": (".llms.openai.openai", "MistralEmbeddingConfig"), - "OpenAIImageVariationConfig": ( - ".llms.openai.image_variations.transformation", - "OpenAIImageVariationConfig", - ), - "DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"), - "ClfAiGatewayConfig": ( - ".llms.clf_ai_gateway.chat.transformation", - "ClfAiGatewayConfig", - ), - "DeepgramAudioTranscriptionConfig": ( - ".llms.deepgram.audio_transcription.transformation", - "DeepgramAudioTranscriptionConfig", - ), - "TopazImageVariationConfig": ( - ".llms.topaz.image_variations.transformation", - "TopazImageVariationConfig", - ), - "OpenAITextCompletionConfig": ( - "litellm.llms.openai.completion.transformation", - "OpenAITextCompletionConfig", - ), - "GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"), - "BedrockOpenAIResponsesConfig": ( - ".llms.bedrock.responses.transformation", - "BedrockOpenAIResponsesConfig", - ), - "BedrockMantleChatConfig": ( - ".llms.bedrock_mantle.chat.transformation", - "BedrockMantleChatConfig", - ), - "A2AConfig": (".llms.a2a.chat.transformation", "A2AConfig"), - "GenAIHubOrchestrationConfig": ( - ".llms.sap.chat.transformation", - "GenAIHubOrchestrationConfig", - ), - "VoyageEmbeddingConfig": ( - ".llms.voyage.embedding.transformation", - "VoyageEmbeddingConfig", - ), - "VoyageContextualEmbeddingConfig": ( - ".llms.voyage.embedding.transformation_contextual", - "VoyageContextualEmbeddingConfig", - ), - "VoyageMultimodalEmbeddingConfig": ( - ".llms.voyage.embedding.transformation_multimodal", - "VoyageMultimodalEmbeddingConfig", - ), - "InfinityEmbeddingConfig": ( - ".llms.infinity.embedding.transformation", - "InfinityEmbeddingConfig", - ), - "PerplexityEmbeddingConfig": ( - ".llms.perplexity.embedding.transformation", - "PerplexityEmbeddingConfig", - ), - "AzureAIStudioConfig": ( - ".llms.azure_ai.chat.transformation", - "AzureAIStudioConfig", - ), - "MistralConfig": (".llms.mistral.chat.transformation", "MistralConfig"), - "OpenAIResponsesAPIConfig": ( - ".llms.openai.responses.transformation", - "OpenAIResponsesAPIConfig", - ), - "AzureOpenAIResponsesAPIConfig": ( - ".llms.azure.responses.transformation", - "AzureOpenAIResponsesAPIConfig", - ), - "AzureOpenAIOSeriesResponsesAPIConfig": ( - ".llms.azure.responses.o_series_transformation", - "AzureOpenAIOSeriesResponsesAPIConfig", - ), - "AzureAIResponsesAPIConfig": ( - ".llms.azure_ai.responses.transformation", - "AzureAIResponsesAPIConfig", - ), - "XAIResponsesAPIConfig": ( - ".llms.xai.responses.transformation", - "XAIResponsesAPIConfig", - ), - "LiteLLMProxyResponsesAPIConfig": ( - ".llms.litellm_proxy.responses.transformation", - "LiteLLMProxyResponsesAPIConfig", - ), - "HostedVLLMResponsesAPIConfig": ( - ".llms.hosted_vllm.responses.transformation", - "HostedVLLMResponsesAPIConfig", - ), - "FireworksAIResponsesAPIConfig": ( - ".llms.fireworks_ai.responses.transformation", - "FireworksAIResponsesAPIConfig", - ), - "VolcEngineResponsesAPIConfig": ( - ".llms.volcengine.responses.transformation", - "VolcEngineResponsesAPIConfig", - ), - "ManusResponsesAPIConfig": ( - ".llms.manus.responses.transformation", - "ManusResponsesAPIConfig", - ), - "PerplexityResponsesConfig": ( - ".llms.perplexity.responses.transformation", - "PerplexityResponsesConfig", - ), - "DatabricksResponsesAPIConfig": ( - ".llms.databricks.responses.transformation", - "DatabricksResponsesAPIConfig", - ), - "OpenRouterResponsesAPIConfig": ( - ".llms.openrouter.responses.transformation", - "OpenRouterResponsesAPIConfig", - ), - "BedrockMantleResponsesAPIConfig": ( - ".llms.bedrock_mantle.responses.transformation", - "BedrockMantleResponsesAPIConfig", - ), - "GoogleAIStudioInteractionsConfig": ( - ".llms.gemini.interactions.transformation", - "GoogleAIStudioInteractionsConfig", - ), - "VertexAIInteractionsConfig": ( - ".llms.vertex_ai.interactions.transformation", - "VertexAIInteractionsConfig", - ), - "OpenAIOSeriesConfig": ( - ".llms.openai.chat.o_series_transformation", - "OpenAIOSeriesConfig", - ), - "AnthropicSkillsConfig": ( - ".llms.anthropic.skills.transformation", - "AnthropicSkillsConfig", - ), - "BaseSkillsAPIConfig": ( - ".llms.base_llm.skills.transformation", - "BaseSkillsAPIConfig", - ), - "GradientAIConfig": (".llms.gradient_ai.chat.transformation", "GradientAIConfig"), - # Alias for backwards compatibility - "OpenAIO1Config": ( - ".llms.openai.chat.o_series_transformation", - "OpenAIOSeriesConfig", - ), # Alias - "OpenAIGPTConfig": (".llms.openai.chat.gpt_transformation", "OpenAIGPTConfig"), - "OpenAIGPT5Config": (".llms.openai.chat.gpt_reasoning_transformation", "OpenAIGPT5Config"), - "OpenAIWhisperAudioTranscriptionConfig": ( - ".llms.openai.transcriptions.whisper_transformation", - "OpenAIWhisperAudioTranscriptionConfig", - ), - "OpenAIGPTAudioTranscriptionConfig": ( - ".llms.openai.transcriptions.gpt_transformation", - "OpenAIGPTAudioTranscriptionConfig", - ), - "OpenAIGPTAudioConfig": ( - ".llms.openai.chat.gpt_audio_transformation", - "OpenAIGPTAudioConfig", - ), - "NvidiaNimConfig": (".llms.nvidia_nim.chat.transformation", "NvidiaNimConfig"), - "NvidiaNimEmbeddingConfig": (".llms.nvidia_nim.embed", "NvidiaNimEmbeddingConfig"), - "FeatherlessAIConfig": ( - ".llms.featherless_ai.chat.transformation", - "FeatherlessAIConfig", - ), - "ScaleDownChatConfig": ( - ".llms.scaledown.chat.transformation", - "ScaleDownChatConfig", - ), - "CerebrasConfig": (".llms.cerebras.chat", "CerebrasConfig"), - "NadirConfig": (".llms.nadir.chat.transformation", "NadirConfig"), - "BasetenConfig": (".llms.baseten.chat", "BasetenConfig"), - "SambanovaConfig": (".llms.sambanova.chat", "SambanovaConfig"), - "SambaNovaEmbeddingConfig": ( - ".llms.sambanova.embedding.transformation", - "SambaNovaEmbeddingConfig", - ), - "FireworksAIConfig": ( - ".llms.fireworks_ai.chat.transformation", - "FireworksAIConfig", - ), - "FireworksAITextCompletionConfig": ( - ".llms.fireworks_ai.completion.transformation", - "FireworksAITextCompletionConfig", - ), - "FireworksAIEmbeddingConfig": ( - ".llms.fireworks_ai.embed.fireworks_ai_transformation", - "FireworksAIEmbeddingConfig", - ), - "FriendliaiChatConfig": ( - ".llms.friendliai.chat.transformation", - "FriendliaiChatConfig", - ), - "JinaAIEmbeddingConfig": ( - ".llms.jina_ai.embedding.transformation", - "JinaAIEmbeddingConfig", - ), - "XAIChatConfig": (".llms.xai.chat.transformation", "XAIChatConfig"), - "ZAIChatConfig": (".llms.zai.chat.transformation", "ZAIChatConfig"), - "AIMLChatConfig": (".llms.aiml.chat.transformation", "AIMLChatConfig"), - "VolcEngineChatConfig": ( - ".llms.volcengine.chat.transformation", - "VolcEngineChatConfig", - ), - "CodestralTextCompletionConfig": ( - ".llms.codestral.completion.transformation", - "CodestralTextCompletionConfig", - ), - "InceptionTextCompletionConfig": ( - ".llms.inception.completion.transformation", - "InceptionTextCompletionConfig", - ), - "AzureOpenAIAssistantsAPIConfig": ( - ".llms.azure.azure", - "AzureOpenAIAssistantsAPIConfig", - ), - "HerokuChatConfig": (".llms.heroku.chat.transformation", "HerokuChatConfig"), - "CometAPIConfig": (".llms.cometapi.chat.transformation", "CometAPIConfig"), - "AzureOpenAIConfig": (".llms.azure.chat.gpt_transformation", "AzureOpenAIConfig"), - "AzureOpenAIGPT5Config": ( - ".llms.azure.chat.gpt_reasoning_transformation", - "AzureOpenAIGPT5Config", - ), - "AzureOpenAITextConfig": ( - ".llms.azure.completion.transformation", - "AzureOpenAITextConfig", - ), - "AzureSpeechAudioTranscriptionConfig": ( - ".llms.azure.audio_transcription.transformation", - "AzureSpeechAudioTranscriptionConfig", - ), - "HostedVLLMChatConfig": ( - ".llms.hosted_vllm.chat.transformation", - "HostedVLLMChatConfig", - ), - "HostedVLLMEmbeddingConfig": ( - ".llms.hosted_vllm.embedding.transformation", - "HostedVLLMEmbeddingConfig", - ), - # Alias for backwards compatibility - "VolcEngineConfig": ( - ".llms.volcengine.chat.transformation", - "VolcEngineChatConfig", - ), # Alias - "LlamafileChatConfig": ( - ".llms.llamafile.chat.transformation", - "LlamafileChatConfig", - ), - "LiteLLMProxyChatConfig": ( - ".llms.litellm_proxy.chat.transformation", - "LiteLLMProxyChatConfig", - ), - "VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"), - "DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"), - "TencentChatConfig": (".llms.tencent.chat.transformation", "TencentChatConfig"), - "LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"), - "LmStudioEmbeddingConfig": ( - ".llms.lm_studio.embed.transformation", - "LmStudioEmbeddingConfig", - ), - "NscaleConfig": (".llms.nscale.chat.transformation", "NscaleConfig"), - "PerplexityChatConfig": ( - ".llms.perplexity.chat.transformation", - "PerplexityChatConfig", - ), - "AzureOpenAIO1Config": ( - ".llms.azure.chat.o_series_transformation", - "AzureOpenAIO1Config", - ), - "IBMWatsonXAIConfig": ( - ".llms.watsonx.completion.transformation", - "IBMWatsonXAIConfig", - ), - "IBMWatsonXChatConfig": ( - ".llms.watsonx.chat.transformation", - "IBMWatsonXChatConfig", - ), - "IBMWatsonXEmbeddingConfig": ( - ".llms.watsonx.embed.transformation", - "IBMWatsonXEmbeddingConfig", - ), - "GenAIHubEmbeddingConfig": ( - ".llms.sap.embed.transformation", - "GenAIHubEmbeddingConfig", - ), - "IBMWatsonXAudioTranscriptionConfig": ( - ".llms.watsonx.audio_transcription.transformation", - "IBMWatsonXAudioTranscriptionConfig", - ), - "GithubCopilotConfig": ( - ".llms.github_copilot.chat.transformation", - "GithubCopilotConfig", - ), - "GithubCopilotResponsesAPIConfig": ( - ".llms.github_copilot.responses.transformation", - "GithubCopilotResponsesAPIConfig", - ), - "GithubCopilotEmbeddingConfig": ( - ".llms.github_copilot.embedding.transformation", - "GithubCopilotEmbeddingConfig", - ), - "ChatGPTConfig": (".llms.chatgpt.chat.transformation", "ChatGPTConfig"), - "ChatGPTResponsesAPIConfig": ( - ".llms.chatgpt.responses.transformation", - "ChatGPTResponsesAPIConfig", - ), - "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"), - "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"), - "GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"), - "GigaChatEmbeddingConfig": ( - ".llms.gigachat.embedding.transformation", - "GigaChatEmbeddingConfig", - ), - "DashScopeChatConfig": ( - ".llms.dashscope.chat.transformation", - "DashScopeChatConfig", - ), - "QwenCloudChatConfig": ( - ".llms.dashscope.qwencloud", - "QwenCloudChatConfig", - ), - "QwenAIPlatformChatConfig": ( - ".llms.dashscope.qwen_ai_platform", - "QwenAIPlatformChatConfig", - ), - "GDCGeminiConfig": ( - ".llms.gdc.chat.transformation", - "GDCGeminiConfig", - ), - "ModelScopeChatConfig": ( - ".llms.modelscope.chat.transformation", - "ModelScopeChatConfig", - ), - "MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"), - "DockerModelRunnerChatConfig": ( - ".llms.docker_model_runner.chat.transformation", - "DockerModelRunnerChatConfig", - ), - "V0ChatConfig": (".llms.v0.chat.transformation", "V0ChatConfig"), - "OCIChatConfig": (".llms.oci.chat.transformation", "OCIChatConfig"), - "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), - "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), - "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), - "InceptionChatConfig": ( - ".llms.inception.chat.transformation", - "InceptionChatConfig", - ), - "HyperbolicChatConfig": ( - ".llms.hyperbolic.chat.transformation", - "HyperbolicChatConfig", - ), - "VercelAIGatewayConfig": ( - ".llms.vercel_ai_gateway.chat.transformation", - "VercelAIGatewayConfig", - ), - "EdenAIChatConfig": (".llms.edenai.chat.transformation", "EdenAIChatConfig"), - "EdenAIResponsesAPIConfig": (".llms.edenai.responses.transformation", "EdenAIResponsesAPIConfig"), - "EdenAIAnthropicMessagesConfig": (".llms.edenai.messages.transformation", "EdenAIAnthropicMessagesConfig"), - "EdenAIEmbeddingConfig": (".llms.edenai.embedding.transformation", "EdenAIEmbeddingConfig"), - "EdenAIAudioTranscriptionConfig": ( - ".llms.edenai.audio_transcription.transformation", - "EdenAIAudioTranscriptionConfig", - ), - "EdenAITextToSpeechConfig": (".llms.edenai.text_to_speech.transformation", "EdenAITextToSpeechConfig"), - "EdenAIImageGenerationConfig": (".llms.edenai.image_generation.transformation", "EdenAIImageGenerationConfig"), - "EdenAIVideoConfig": (".llms.edenai.videos.transformation", "EdenAIVideoConfig"), - "FalAIChatConfig": (".llms.fal_ai.chat.transformation", "FalAIChatConfig"), - "FalAIError": (".llms.fal_ai.chat.transformation", "FalAIError"), - "OVHCloudChatConfig": (".llms.ovhcloud.chat.transformation", "OVHCloudChatConfig"), - "OVHCloudEmbeddingConfig": ( - ".llms.ovhcloud.embedding.transformation", - "OVHCloudEmbeddingConfig", - ), - "CometAPIEmbeddingConfig": ( - ".llms.cometapi.embed.transformation", - "CometAPIEmbeddingConfig", - ), - "LemonadeChatConfig": (".llms.lemonade.chat.transformation", "LemonadeChatConfig"), - "SnowflakeEmbeddingConfig": ( - ".llms.snowflake.embedding.transformation", - "SnowflakeEmbeddingConfig", - ), - "AmazonNovaChatConfig": ( - ".llms.amazon_nova.chat.transformation", - "AmazonNovaChatConfig", - ), - "SonioxAudioTranscriptionConfig": ( - ".llms.soniox.audio_transcription.transformation", - "SonioxAudioTranscriptionConfig", - ), -} - -# Import map for utils module lazy imports -_UTILS_MODULE_IMPORT_MAP: Final = { - "encoding": ("litellm.main", "encoding"), - "BaseVectorStore": ( - "litellm.integrations.vector_store_integrations.base_vector_store", - "BaseVectorStore", - ), - "CredentialAccessor": ( - "litellm.litellm_core_utils.credential_accessor", - "CredentialAccessor", - ), - "exception_type": ( - "litellm.litellm_core_utils.exception_mapping_utils", - "exception_type", - ), - "get_error_message": ( - "litellm.litellm_core_utils.exception_mapping_utils", - "get_error_message", - ), - "_get_response_headers": ( - "litellm.litellm_core_utils.exception_mapping_utils", - "_get_response_headers", - ), - "get_llm_provider": ( - "litellm.litellm_core_utils.get_llm_provider_logic", - "get_llm_provider", - ), - "_is_non_openai_azure_model": ( - "litellm.litellm_core_utils.get_llm_provider_logic", - "is_non_openai_azure_model", - ), - "is_non_openai_azure_model": ( - "litellm.litellm_core_utils.get_llm_provider_logic", - "is_non_openai_azure_model", - ), - "get_supported_openai_params": ( - "litellm.litellm_core_utils.get_supported_openai_params", - "get_supported_openai_params", - ), - "LiteLLMResponseObjectHandler": ( - "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", - "LiteLLMResponseObjectHandler", - ), - "_handle_invalid_parallel_tool_calls": ( - "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", - "handle_invalid_parallel_tool_calls", - ), - "handle_invalid_parallel_tool_calls": ( - "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", - "handle_invalid_parallel_tool_calls", - ), - "convert_to_model_response_object": ( - "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", - "convert_to_model_response_object", - ), - "convert_to_streaming_response": ( - "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", - "convert_to_streaming_response", - ), - "convert_to_streaming_response_async": ( - "litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response", - "convert_to_streaming_response_async", - ), - "get_api_base": ( - "litellm.litellm_core_utils.llm_response_utils.get_api_base", - "get_api_base", - ), - "ResponseMetadata": ( - "litellm.litellm_core_utils.llm_response_utils.response_metadata", - "ResponseMetadata", - ), - "_parse_content_for_reasoning": ( - "litellm.litellm_core_utils.prompt_templates.common_utils", - "parse_content_for_reasoning", - ), - "parse_content_for_reasoning": ( - "litellm.litellm_core_utils.prompt_templates.common_utils", - "parse_content_for_reasoning", - ), - "LiteLLMLoggingObject": ( - "litellm.litellm_core_utils.redact_messages", - "LiteLLMLoggingObject", - ), - "redact_message_input_output_from_logging": ( - "litellm.litellm_core_utils.redact_messages", - "redact_message_input_output_from_logging", - ), - "CustomStreamWrapper": ( - "litellm.litellm_core_utils.streaming_handler", - "CustomStreamWrapper", - ), - "BaseGoogleGenAIGenerateContentConfig": ( - "litellm.llms.base_llm.google_genai.transformation", - "BaseGoogleGenAIGenerateContentConfig", - ), - "BaseSearchConfig": ( - "litellm.llms.base_llm.search.transformation", - "BaseSearchConfig", - ), - "BaseTextToSpeechConfig": ( - "litellm.llms.base_llm.text_to_speech.transformation", - "BaseTextToSpeechConfig", - ), - "BedrockModelInfo": ("litellm.llms.bedrock.common_utils", "BedrockModelInfo"), - "CohereModelInfo": ("litellm.llms.cohere.common_utils", "CohereModelInfo"), - "Rules": ("litellm.litellm_core_utils.rules", "Rules"), - "AsyncHTTPHandler": ("litellm.llms.custom_httpx.http_handler", "AsyncHTTPHandler"), - "HTTPHandler": ("litellm.llms.custom_httpx.http_handler", "HTTPHandler"), - "get_num_retries_from_retry_policy": ( - "litellm.litellm_core_utils.get_retry_from_policy", - "get_num_retries_from_retry_policy", - ), - "reset_retry_policy": ( - "litellm.litellm_core_utils.get_retry_from_policy", - "reset_retry_policy", - ), - "get_coroutine_checker": ( - "litellm.litellm_core_utils.cached_imports", - "get_coroutine_checker", - ), - "get_litellm_logging_class": ( - "litellm.litellm_core_utils.cached_imports", - "get_litellm_logging_class", - ), - "get_set_callbacks": ( - "litellm.litellm_core_utils.cached_imports", - "get_set_callbacks", - ), - "get_litellm_metadata_from_kwargs": ( - "litellm.litellm_core_utils.core_helpers", - "get_litellm_metadata_from_kwargs", - ), - "map_finish_reason": ( - "litellm.litellm_core_utils.core_helpers", - "map_finish_reason", - ), - "process_response_headers": ( - "litellm.litellm_core_utils.core_helpers", - "process_response_headers", - ), - "delete_nested_value": ( - "litellm.litellm_core_utils.dot_notation_indexing", - "delete_nested_value", - ), - "is_nested_path": ( - "litellm.litellm_core_utils.dot_notation_indexing", - "is_nested_path", - ), - "_get_base_model_from_litellm_call_metadata": ( - "litellm.litellm_core_utils.get_litellm_params", - "get_base_model_from_litellm_call_metadata", - ), - "get_base_model_from_litellm_call_metadata": ( - "litellm.litellm_core_utils.get_litellm_params", - "get_base_model_from_litellm_call_metadata", - ), - "get_litellm_params": ( - "litellm.litellm_core_utils.get_litellm_params", - "get_litellm_params", - ), - "_ensure_extra_body_is_safe": ( - "litellm.litellm_core_utils.llm_request_utils", - "ensure_extra_body_is_safe", - ), - "ensure_extra_body_is_safe": ( - "litellm.litellm_core_utils.llm_request_utils", - "ensure_extra_body_is_safe", - ), - "get_formatted_prompt": ( - "litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt", - "get_formatted_prompt", - ), - "get_response_headers": ( - "litellm.litellm_core_utils.llm_response_utils.get_headers", - "get_response_headers", - ), - "update_response_metadata": ( - "litellm.litellm_core_utils.llm_response_utils.response_metadata", - "update_response_metadata", - ), - "executor": ("litellm.litellm_core_utils.thread_pool_executor", "executor"), - "BaseAnthropicMessagesConfig": ( - "litellm.llms.base_llm.anthropic_messages.transformation", - "BaseAnthropicMessagesConfig", - ), - "BaseAudioTranscriptionConfig": ( - "litellm.llms.base_llm.audio_transcription.transformation", - "BaseAudioTranscriptionConfig", - ), - "BaseBatchesConfig": ( - "litellm.llms.base_llm.batches.transformation", - "BaseBatchesConfig", - ), - "BaseContainerConfig": ( - "litellm.llms.base_llm.containers.transformation", - "BaseContainerConfig", - ), - "BaseEmbeddingConfig": ( - "litellm.llms.base_llm.embedding.transformation", - "BaseEmbeddingConfig", - ), - "BaseImageEditConfig": ( - "litellm.llms.base_llm.image_edit.transformation", - "BaseImageEditConfig", - ), - "BaseImageGenerationConfig": ( - "litellm.llms.base_llm.image_generation.transformation", - "BaseImageGenerationConfig", - ), - "BaseImageVariationConfig": ( - "litellm.llms.base_llm.image_variations.transformation", - "BaseImageVariationConfig", - ), - "BasePassthroughConfig": ( - "litellm.llms.base_llm.passthrough.transformation", - "BasePassthroughConfig", - ), - "BaseRealtimeConfig": ( - "litellm.llms.base_llm.realtime.transformation", - "BaseRealtimeConfig", - ), - "BaseRerankConfig": ( - "litellm.llms.base_llm.rerank.transformation", - "BaseRerankConfig", - ), - "BaseVectorStoreConfig": ( - "litellm.llms.base_llm.vector_store.transformation", - "BaseVectorStoreConfig", - ), - "BaseVectorStoreFilesConfig": ( - "litellm.llms.base_llm.vector_store_files.transformation", - "BaseVectorStoreFilesConfig", - ), - "BaseVideoConfig": ( - "litellm.llms.base_llm.videos.transformation", - "BaseVideoConfig", - ), - "ANTHROPIC_API_ONLY_HEADERS": ( - "litellm.types.llms.anthropic", - "ANTHROPIC_API_ONLY_HEADERS", - ), - "AnthropicThinkingParam": ( - "litellm.types.llms.anthropic", - "AnthropicThinkingParam", - ), - "RerankResponse": ("litellm.types.rerank", "RerankResponse"), - "ChatCompletionDeltaToolCallChunk": ( - "litellm.types.llms.openai", - "ChatCompletionDeltaToolCallChunk", - ), - "ChatCompletionToolCallChunk": ( - "litellm.types.llms.openai", - "ChatCompletionToolCallChunk", - ), - "ChatCompletionToolCallFunctionChunk": ( - "litellm.types.llms.openai", - "ChatCompletionToolCallFunctionChunk", - ), - "LiteLLM_Params": ("litellm.types.router", "LiteLLM_Params"), -} - -# Export all name tuples and import maps for use in _lazy_imports.py -__all__ = [ - "BEDROCK_TYPES_NAMES", - "CACHING_NAMES", - "COST_CALCULATOR_NAMES", - "DOTPROMPT_NAMES", - "HTTP_HANDLER_NAMES", - "LITELLM_LOGGING_NAMES", - "LLM_CLIENT_CACHE_NAMES", - "LLM_CONFIG_NAMES", - "LLM_PROVIDER_LOGIC_NAMES", - "TOKEN_COUNTER_NAMES", - "TYPES_NAMES", - "TYPES_UTILS_NAMES", - "UTILS_MODULE_NAMES", - "UTILS_NAMES", - "_BEDROCK_TYPES_IMPORT_MAP", - "_CACHING_IMPORT_MAP", - "_COST_CALCULATOR_IMPORT_MAP", - "_DOTPROMPT_IMPORT_MAP", - "_LITELLM_LOGGING_IMPORT_MAP", - "_LLM_CONFIGS_IMPORT_MAP", - "_LLM_PROVIDER_LOGIC_IMPORT_MAP", - "_TOKEN_COUNTER_IMPORT_MAP", - "_TYPES_IMPORT_MAP", - "_TYPES_UTILS_IMPORT_MAP", - "_UTILS_IMPORT_MAP", - "_UTILS_MODULE_IMPORT_MAP", -] diff --git a/litellm/utils.py b/litellm/utils.py index 5ec089a106a..c7f3d4fb357 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -12025,15 +12025,6 @@ def should_run_mock_completion( def __getattr__(name: str) -> Any: - """Lazy import handler for utils module with cached registry for improved performance.""" - # Use cached registry from _lazy_imports instead of importing tuples every time - from litellm._lazy_imports import get_lazy_import_registry + from litellm._lazy_imports import resolve_lazy_attribute - registry: Final = get_lazy_import_registry() - - # Check if name is in registry and call the cached handler function - if name in registry: - handler_func: Final = registry[name] - return handler_func(name) - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + return resolve_lazy_attribute(__name__, name) diff --git a/tests/unit/test_lazy_imports.py b/tests/unit/test_lazy_imports.py index 43aee4616d2..1f39bdede5a 100644 --- a/tests/unit/test_lazy_imports.py +++ b/tests/unit/test_lazy_imports.py @@ -1,5 +1,4 @@ -"""Simple tests for lazy import functionality.""" - +import importlib import os import subprocess import sys @@ -7,38 +6,9 @@ from typing import Final import pytest - import litellm -from litellm._lazy_imports import ( - COST_CALCULATOR_NAMES, - LITELLM_LOGGING_NAMES, - UTILS_NAMES, - TOKEN_COUNTER_NAMES, - CACHING_NAMES, - BEDROCK_TYPES_NAMES, - TYPES_UTILS_NAMES, - LLM_CLIENT_CACHE_NAMES, - HTTP_HANDLER_NAMES, - _lazy_import_cost_calculator, - _lazy_import_litellm_logging, - _lazy_import_utils, - _lazy_import_token_counter, - _lazy_import_bedrock_types, - _lazy_import_types_utils, - _lazy_import_caching, - _lazy_import_llm_client_cache, - _lazy_import_http_handlers, - DOTPROMPT_NAMES, - _lazy_import_dotprompt, - LLM_CONFIG_NAMES, - _lazy_import_llm_configs, - TYPES_NAMES, - _lazy_import_types, - LLM_PROVIDER_LOGIC_NAMES, - _lazy_import_llm_provider_logic, - UTILS_MODULE_NAMES, - _lazy_import_utils_module, -) +import litellm.utils +from litellm._lazy_imports_registry import LITELLM_ATTRIBUTES, UTILS_ATTRIBUTES from tests.test_litellm_rust.support.child_interpreter import run_child_interpreter @@ -58,314 +28,75 @@ def test_import_litellm_does_not_load_fastapi_or_bpe_table(): assert result.stdout.strip() == "" -def _clear_names_from_globals(names: tuple): - """Clear all names from litellm globals.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - for name in names: - if name in litellm_globals: - del litellm_globals[name] - - -def _clear_names_from_utils_globals(names: tuple): - """Clear all names from litellm.utils globals.""" - # Get the actual globals dict, not a copy - utils_globals = sys.modules["litellm.utils"].__dict__ - for name in names: - if name in utils_globals: - del utils_globals[name] - - -def _verify_only_requested_name_imported(name: str, all_names: tuple): - """Verify that only the requested name is in globals, not the others.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - for other_name in all_names: - if other_name != name: - assert ( - other_name not in litellm_globals - ), f"{other_name} should not be imported when importing {name}" - - -def _verify_only_requested_name_imported_in_utils(name: str, all_names: tuple): - """Verify that only the requested name is in utils globals, not the others.""" - # Get the actual globals dict, not a copy - utils_globals = sys.modules["litellm.utils"].__dict__ - for other_name in all_names: - if other_name != name: - assert ( - other_name not in utils_globals - ), f"{other_name} should not be imported when importing {name}" - - -def test_cost_calculator_lazy_imports(): - """Test that all cost calculator functions can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - # Test each name individually - only that name should be imported - for name in COST_CALCULATOR_NAMES: - # Clear all names before importing just one - _clear_names_from_globals(COST_CALCULATOR_NAMES) - - func = _lazy_import_cost_calculator(name) - assert func is not None - assert callable(func) - assert name in litellm_globals - - # Verify only the requested name is in globals, not the others - _verify_only_requested_name_imported(name, COST_CALCULATOR_NAMES) - - -def test_litellm_logging_lazy_imports(): - """Test that all litellm_logging items can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - # Test each name individually - only that name should be imported - for name in LITELLM_LOGGING_NAMES: - # Clear all names before importing just one - _clear_names_from_globals(LITELLM_LOGGING_NAMES) - - item = _lazy_import_litellm_logging(name) - assert item is not None - assert name in litellm_globals - - # Verify only the requested name is in globals, not the others - _verify_only_requested_name_imported(name, LITELLM_LOGGING_NAMES) - - -def test_utils_lazy_imports(): - """Test that all utils functions can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - # Test each name individually - only that name should be imported - for name in UTILS_NAMES: - # Clear all names before importing just one - _clear_names_from_globals(UTILS_NAMES) - - attr = _lazy_import_utils(name) - assert attr is not None - assert name in litellm_globals - - # Verify only the requested name is in globals, not the others - _verify_only_requested_name_imported(name, UTILS_NAMES) - - -def test_caching_lazy_imports(): - """Test that all caching classes can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - # Test each name individually - only that name should be imported - for name in CACHING_NAMES: - # Clear all names before importing just one - _clear_names_from_globals(CACHING_NAMES) - - cls = _lazy_import_caching(name) - assert cls is not None - assert name in litellm_globals - - # Verify only the requested name is in globals, not the others - _verify_only_requested_name_imported(name, CACHING_NAMES) - - -def test_token_counter_lazy_imports(): - """Test that token counter utilities can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in TOKEN_COUNTER_NAMES: - _clear_names_from_globals(TOKEN_COUNTER_NAMES) - - func = _lazy_import_token_counter(name) - assert func is not None - assert name in litellm_globals - - _verify_only_requested_name_imported(name, TOKEN_COUNTER_NAMES) - - -def test_bedrock_types_lazy_imports(): - """Test that Bedrock type aliases can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in BEDROCK_TYPES_NAMES: - _clear_names_from_globals(BEDROCK_TYPES_NAMES) - - alias = _lazy_import_bedrock_types(name) - assert alias is not None - assert name in litellm_globals - - _verify_only_requested_name_imported(name, BEDROCK_TYPES_NAMES) - - -def test_types_utils_lazy_imports(): - """Test that common types.utils symbols can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in TYPES_UTILS_NAMES: - _clear_names_from_globals(TYPES_UTILS_NAMES) - - obj = _lazy_import_types_utils(name) - assert obj is not None - assert name in litellm_globals - - _verify_only_requested_name_imported(name, TYPES_UTILS_NAMES) - - -def test_llm_client_cache_lazy_imports(): - """Test that LLM client cache class and singleton can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in LLM_CLIENT_CACHE_NAMES: - _clear_names_from_globals(LLM_CLIENT_CACHE_NAMES) - - obj = _lazy_import_llm_client_cache(name) - assert obj is not None - assert name in litellm_globals - - _verify_only_requested_name_imported(name, LLM_CLIENT_CACHE_NAMES) - - -def test_http_handler_lazy_imports(): - """Test that HTTP handler singletons can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in HTTP_HANDLER_NAMES: - _clear_names_from_globals(HTTP_HANDLER_NAMES) - - handler = _lazy_import_http_handlers(name) - assert handler is not None - assert name in litellm_globals - - _verify_only_requested_name_imported(name, HTTP_HANDLER_NAMES) - - -def test_dotprompt_lazy_imports(): - """Test that dotprompt globals can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in DOTPROMPT_NAMES: - _clear_names_from_globals(DOTPROMPT_NAMES) - - obj = _lazy_import_dotprompt(name) - assert name in litellm_globals - - # Only the setter must be callable; others may be None by default - if name == "set_global_prompt_directory": - assert callable(obj), f"{name} should be callable" - - _verify_only_requested_name_imported(name, DOTPROMPT_NAMES) - - -def test_unknown_attribute_raises_error(): - """Test that unknown attributes raise AttributeError.""" - with pytest.raises(AttributeError): - _lazy_import_cost_calculator("unknown") - - with pytest.raises(AttributeError): - _lazy_import_litellm_logging("unknown") - - with pytest.raises(AttributeError): - _lazy_import_utils("unknown") - - with pytest.raises(AttributeError): - _lazy_import_caching("unknown") - - with pytest.raises(AttributeError): - _lazy_import_token_counter("unknown") - - with pytest.raises(AttributeError): - _lazy_import_llm_client_cache("unknown") - - with pytest.raises(AttributeError): - _lazy_import_bedrock_types("unknown") - - with pytest.raises(AttributeError): - _lazy_import_types_utils("unknown") - - with pytest.raises(AttributeError): - _lazy_import_llm_configs("unknown") - - with pytest.raises(AttributeError): - _lazy_import_types("unknown") - - with pytest.raises(AttributeError): - _lazy_import_llm_provider_logic("unknown") - - with pytest.raises(AttributeError): - _lazy_import_utils_module("unknown") - - -def test_llm_config_lazy_imports(): - """Test that LLM config classes can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in LLM_CONFIG_NAMES: - _clear_names_from_globals(LLM_CONFIG_NAMES) - - obj = _lazy_import_llm_configs(name) - assert obj is not None - assert name in litellm_globals - # Config classes should be classes/types - assert isinstance(obj, type), f"{name} should be a class" - - _verify_only_requested_name_imported(name, LLM_CONFIG_NAMES) - - -def test_types_lazy_imports(): - """Test that type classes can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in TYPES_NAMES: - _clear_names_from_globals(TYPES_NAMES) - - obj = _lazy_import_types(name) - assert obj is not None - assert name in litellm_globals - # Type classes should be classes/types - assert isinstance(obj, type), f"{name} should be a class" - - _verify_only_requested_name_imported(name, TYPES_NAMES) - - -def test_llm_provider_logic_lazy_imports(): - """Test that LLM provider logic functions can be lazy imported.""" - # Get the actual globals dict, not a copy - litellm_globals = sys.modules["litellm"].__dict__ - - for name in LLM_PROVIDER_LOGIC_NAMES: - _clear_names_from_globals(LLM_PROVIDER_LOGIC_NAMES) - - func = _lazy_import_llm_provider_logic(name) - assert func is not None - assert callable(func) - assert name in litellm_globals - - _verify_only_requested_name_imported(name, LLM_PROVIDER_LOGIC_NAMES) - - -def test_utils_module_lazy_imports(): - """Test that utils module attributes can be lazy imported.""" - # Get the actual globals dict, not a copy - utils_globals = sys.modules["litellm.utils"].__dict__ - - for name in UTILS_MODULE_NAMES: - _clear_names_from_utils_globals(UTILS_MODULE_NAMES) - - obj = _lazy_import_utils_module(name) - assert obj is not None - assert name in utils_globals - - _verify_only_requested_name_imported_in_utils(name, UTILS_MODULE_NAMES) +def _declared(location: tuple[str, str]) -> object: + module_path, attr_name = location + return getattr(importlib.import_module(module_path, package="litellm"), attr_name) + + +@pytest.mark.parametrize("name", sorted(LITELLM_ATTRIBUTES)) +def test_litellm_attribute_is_the_declared_object(name: str): + assert getattr(litellm, name) is _declared(LITELLM_ATTRIBUTES[name]) + + +@pytest.mark.parametrize("name", sorted(UTILS_ATTRIBUTES)) +def test_utils_attribute_is_the_same_object_from_litellm_and_litellm_utils(name: str): + expected: Final = _declared(UTILS_ATTRIBUTES[name]) + assert getattr(litellm.utils, name) is expected + assert getattr(litellm, name) is expected + + +@pytest.mark.parametrize( + ("instance_name", "class_name"), + [ + ("in_memory_llm_clients_cache", "LLMClientCache"), + ("openaiOSeriesConfig", "OpenAIOSeriesConfig"), + ("openAIGPTConfig", "OpenAIGPTConfig"), + ("openAIGPTAudioConfig", "OpenAIGPTAudioConfig"), + ("openAIGPT5Config", "OpenAIGPT5Config"), + ("nvidiaNimConfig", "NvidiaNimConfig"), + ("nvidiaNimEmbeddingConfig", "NvidiaNimEmbeddingConfig"), + ("priority_reservation_settings", "PriorityReservationSettings"), + ("logging_callback_manager", "LoggingCallbackManager"), + ], +) +def test_lazy_instance_is_a_single_instance_of_its_class(instance_name: str, class_name: str): + instance: Final = getattr(litellm, instance_name) + assert type(instance) is getattr(litellm, class_name) + assert getattr(litellm, instance_name) is instance + + +def test_module_level_http_clients_use_request_timeout(monkeypatch: pytest.MonkeyPatch): + client_names: Final = ("module_level_client", "module_level_aclient") + monkeypatch.setattr(litellm, "request_timeout", 7) + for client_name in client_names: + monkeypatch.delitem(vars(litellm), client_name, raising=False) + try: + assert litellm.module_level_client.timeout == 7 + assert litellm.module_level_aclient.timeout == 7 + assert litellm.module_level_client is litellm.module_level_client + finally: + for client_name in client_names: + vars(litellm).pop(client_name, None) + + +def test_provider_list_lists_every_provider(): + assert litellm.provider_list == list(litellm.LlmProviders) + + +@pytest.mark.parametrize("name", ["proxy", "guardrails", "harness", "_service_logger"]) +def test_submodule_attribute_is_the_submodule(name: str): + assert getattr(litellm, name) is importlib.import_module(f"litellm.{name}") + + +@pytest.mark.parametrize("module", [litellm, litellm.utils]) +def test_unknown_attribute_raises_attribute_error(module: object): + with pytest.raises(AttributeError, match="not_a_litellm_attribute"): + getattr(module, "not_a_litellm_attribute") + assert not hasattr(module, "not_a_litellm_attribute") + + +def test_litellm_instances_are_not_attributes_of_litellm_utils(): + assert not hasattr(litellm.utils, "module_level_client") @pytest.mark.parametrize( diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index d257866b654..fb42f6229e0 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -1185,7 +1185,6 @@ ALIAS_CASES: Final = ( ("litellm", "", "_turn_on_debug", "turn_on_debug", False), ("litellm", "", "_turn_on_json", "turn_on_json", False), ("litellm._lazy_imports", "", "_get_default_encoding", "get_default_encoding", False), - ("litellm._lazy_imports", "", "_get_lazy_import_registry", "get_lazy_import_registry", False), ("litellm._lazy_imports", "", "_get_messages_reach_token_count", "get_messages_reach_token_count", False), ("litellm._lazy_imports", "", "_get_token_counter_new", "get_token_counter_new", False), ("litellm._logging", "", "_is_debugging_on", "is_debugging_on", False), From 571cbca3b0ccdf113e70943f12d2470da85fc2da Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 09:55:48 -0800 Subject: [PATCH 03/17] fix(cost-map): sync OpenAI fine-tune and Codestral 2405 shutdown dates (#50529) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 10 +++++++--- model_prices_and_context_window.json | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0c55e3e45bf..3346fb66ecc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -17539,6 +17539,8 @@ "supports_reasoning": true }, "codestral/codestral-2405": { + "deprecation_date": "2025-06-16", + "deprecation_source": "https://docs.mistral.ai/getting-started/models/", "input_cost_per_token": 0.0, "litellm_provider": "codestral", "max_input_tokens": 32000, @@ -29341,7 +29343,7 @@ "source": "https://api.friendli.ai/serverless/v1/models" }, "ft:babbage-002": { - "deprecation_date": "2026-10-23", + "deprecation_date": "2026-09-28", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.6e-06, @@ -29355,7 +29357,7 @@ "source": "https://developers.openai.com/api/docs/pricing" }, "ft:davinci-002": { - "deprecation_date": "2026-10-23", + "deprecation_date": "2026-09-28", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.2e-05, @@ -29414,7 +29416,7 @@ "supports_tool_choice": true }, "ft:gpt-3.5-turbo-1106": { - "deprecation_date": "2026-10-23", + "deprecation_date": "2026-09-28", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, @@ -49673,6 +49675,8 @@ "mode": "search" }, "text-completion-codestral/codestral-2405": { + "deprecation_date": "2025-06-16", + "deprecation_source": "https://docs.mistral.ai/getting-started/models/", "input_cost_per_token": 0.0, "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0c55e3e45bf..3346fb66ecc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -17539,6 +17539,8 @@ "supports_reasoning": true }, "codestral/codestral-2405": { + "deprecation_date": "2025-06-16", + "deprecation_source": "https://docs.mistral.ai/getting-started/models/", "input_cost_per_token": 0.0, "litellm_provider": "codestral", "max_input_tokens": 32000, @@ -29341,7 +29343,7 @@ "source": "https://api.friendli.ai/serverless/v1/models" }, "ft:babbage-002": { - "deprecation_date": "2026-10-23", + "deprecation_date": "2026-09-28", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.6e-06, @@ -29355,7 +29357,7 @@ "source": "https://developers.openai.com/api/docs/pricing" }, "ft:davinci-002": { - "deprecation_date": "2026-10-23", + "deprecation_date": "2026-09-28", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "successor_model": "gpt-5.6-terra", "input_cost_per_token": 1.2e-05, @@ -29414,7 +29416,7 @@ "supports_tool_choice": true }, "ft:gpt-3.5-turbo-1106": { - "deprecation_date": "2026-10-23", + "deprecation_date": "2026-09-28", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", "successor_model": "gpt-5.6-terra", "input_cost_per_token": 3e-06, @@ -49673,6 +49675,8 @@ "mode": "search" }, "text-completion-codestral/codestral-2405": { + "deprecation_date": "2025-06-16", + "deprecation_source": "https://docs.mistral.ai/getting-started/models/", "input_cost_per_token": 0.0, "litellm_provider": "text-completion-codestral", "max_input_tokens": 32000, From 3a170ca1fb7237c4981f212bf5363154195b5d6e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 7 Nov 2026 10:22:12 -0800 Subject: [PATCH 04/17] fix(utils): drop Anthropic-only output_config under drop_params on providers that do not read it (#50542) --- litellm/utils.py | 11 ++++ tests/unit/test_utils.py | 119 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index c7f3d4fb357..28e2440fbdd 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5474,6 +5474,10 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl _DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") +_DROP_UNREAD_OUTPUT_CONFIG_WARNING: Final = ( + "Dropping `output_config` for model=%s (drop_params=True) because the %s chat transformation does not read it." +) + def _turns_reasoning_off(value: object) -> bool: match value: @@ -6145,6 +6149,13 @@ def get_optional_params( drop_params=True, custom_llm_provider=custom_llm_provider or None, ) + if ( + "output_config" in passed_params + and not _declared_reasoning_efforts(model_info) + and not (isinstance(provider_config, litellm.AnthropicConfig) or "output_config" in supported_params) + ): + litellm.verbose_logger.warning(_DROP_UNREAD_OUTPUT_CONFIG_WARNING, model, custom_llm_provider) + passed_params.pop("output_config") # if user passed in non-default kwargs for specific providers/models, pass them along optional_params = add_provider_specific_params_to_optional_params( optional_params=optional_params, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 700307f48f5..59520938667 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -18,6 +18,7 @@ from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone from functools import partial from pathlib import Path, PurePath +from types import MappingProxyType from typing import Final, SupportsIndex, cast from unittest import mock from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -3871,6 +3872,124 @@ class TestGetOptionalParamsSpeedGate: assert result.get("speed") == "fast" +class TestGetOptionalParamsOutputConfigGate: + """Regression for the issue 24549 follow-up (SLP-2948): Anthropic's ``output_config`` is not a named + OpenAI param, so it rode the provider-specific pass-through into ``extra_body`` and ``drop_params`` never + saw it.""" + + UNREAD_LEGS: Final = ( + ("openai", "gpt-5.6"), + ("azure", "gpt-5.6"), + ("xiaomi_mimo", "mimo-v2.6-pro"), + ("hosted_vllm", "my-model"), + ("gemini", "gemini-3-pro-preview"), + ("bedrock", "us.amazon.nova-pro-v1:0"), + ) + CLAUDE_LEGS: Final = ( + ("anthropic", "claude-opus-4-8"), + ("vertex_ai", "claude-opus-4-8"), + ("azure_ai", "claude-opus-4-8"), + ("bedrock", "converse/anthropic.claude-opus-4-8-v1:0"), + ("bedrock", "invoke/anthropic.claude-opus-4-8-v1:0"), + ) + OUTPUT_CONFIG: Final = MappingProxyType({"effort": "medium"}) + + @staticmethod + def _output_config_anywhere(optional_params: Mapping[str, object]) -> bool: + extra_body: Final = optional_params.get("extra_body") + return "output_config" in optional_params or (isinstance(extra_body, dict) and "output_config" in extra_body) + + @pytest.mark.parametrize("global_drop_params", [True, False]) + @pytest.mark.parametrize(("custom_llm_provider", "model"), UNREAD_LEGS) + def test_drop_params_strips_output_config_on_providers_that_do_not_read_it( + self, monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, model: str, global_drop_params: bool + ) -> None: + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + result: Final = litellm.get_optional_params( + model=model, + custom_llm_provider=custom_llm_provider, + output_config=dict(self.OUTPUT_CONFIG), + drop_params=not global_drop_params, + ) + assert not self._output_config_anywhere(result), result + + def test_drop_params_keeps_the_mapped_reasoning_effort_beside_the_dropped_output_config( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = litellm.get_optional_params( + model="gpt-5.6", + custom_llm_provider="openai", + output_config=dict(self.OUTPUT_CONFIG), + reasoning_effort="low", + drop_params=True, + ) + assert result.get("reasoning_effort") == "low" + assert not self._output_config_anywhere(result), result + + @pytest.mark.parametrize(("custom_llm_provider", "model"), [("openai", "gpt-5.6"), ("hosted_vllm", "my-model")]) + def test_output_config_is_forwarded_untouched_without_drop_params( + self, monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, model: str + ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = litellm.get_optional_params( + model=model, + custom_llm_provider=custom_llm_provider, + output_config=dict(self.OUTPUT_CONFIG), + drop_params=False, + ) + assert self._output_config_anywhere(result), result + + @pytest.mark.parametrize(("custom_llm_provider", "model"), CLAUDE_LEGS) + def test_drop_params_keeps_output_config_for_claude_transformations( + self, monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, model: str + ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = litellm.get_optional_params( + model=model, + custom_llm_provider=custom_llm_provider, + output_config=dict(self.OUTPUT_CONFIG), + drop_params=True, + ) + assert self._output_config_anywhere(result), result + + def test_drop_params_keeps_output_config_for_a_downstream_litellm_proxy( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = litellm.get_optional_params( + model="claude-opus-4-8", + custom_llm_provider="litellm_proxy", + output_config=dict(self.OUTPUT_CONFIG), + drop_params=True, + ) + assert self._output_config_anywhere(result), result + + def test_drop_params_keeps_output_config_for_a_deployment_that_declares_reasoning_effort_levels( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = litellm.get_optional_params( + model="my-kimi", + custom_llm_provider="hosted_vllm", + output_config=dict(self.OUTPUT_CONFIG), + model_info={"id": "kimi-vllm-1", "reasoning_effort_levels": ["low", "medium", "high"]}, + drop_params=True, + ) + assert self._output_config_anywhere(result), result + + def test_allowed_openai_params_overrides_the_drop(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + result: Final = litellm.get_optional_params( + model="gpt-5.6", + custom_llm_provider="openai", + output_config=dict(self.OUTPUT_CONFIG), + allowed_openai_params=["output_config"], + drop_params=True, + ) + assert self._output_config_anywhere(result), result + + class TestGetOptionalParamsDeepSeek: """Tests that deepseek provider uses DeepSeekChatConfig for parameter mapping.""" From ffc8d6cf66cb4c56b60a3175828d8cc4012e6386 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 7 Nov 2026 10:34:04 -0800 Subject: [PATCH 05/17] feat(cost-map): read alias_of pointer rows in the catalog and the loader (#50512) * feat(cost-map): read alias_of pointer rows in the catalog and the loader Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): keep alias_of rows out of alias lists and add them to the registry schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): let alias lists win over alias_of rows in both loaders and never serve a malformed pointer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/model-catalog/src/catalog.rs | 87 ++++++++++++++++--- .../crates/model-catalog/src/error.rs | 10 +++ .../crates/model-catalog/src/schema.rs | 12 ++- .../model-catalog/src/wire/attributes.rs | 2 + .../crates/model-catalog/tests/catalog.rs | 79 +++++++++++++++++ .../crates/model-catalog/tests/schema.rs | 4 + .../litellm_core_utils/get_model_cost_map.py | 37 ++++++-- .../test_get_model_cost_map.py | 39 +++++++++ tests/unit/test_utils.py | 2 + 9 files changed, 252 insertions(+), 20 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index 98af785b17e..55a95416da3 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -142,6 +142,7 @@ impl Catalog { let mut sections = Vec::new(); let mut entries = IndexMap::with_capacity(document.len()); let mut alias_lists = Vec::new(); + let mut pointer_rows = Vec::new(); for (index, (name, value)) in document.into_iter().enumerate() { let fields = match value { Value::Object(fields) if !is_section(&name) => fields, @@ -157,16 +158,34 @@ impl Catalog { continue; } }; - let (model, aliases, row_problems) = read_row(&fields); + let row = read_row(&fields); + let readable = row.problems.is_empty(); problems.extend( - row_problems + row.problems .into_iter() .map(|problem| problem.prepend(&name)), ); - if let Some(aliases) = aliases { + if let Some(target) = row.alias_of { + if readable { + pointer_rows.push((name.clone(), target)); + } + sections.push(Section { + index, + key: name, + value: Value::Object(fields), + }); + continue; + } + if let Some(aliases) = row.aliases { alias_lists.push((name.clone(), aliases)); } - entries.insert(name, ModelEntry { fields, model }); + entries.insert( + name, + ModelEntry { + fields, + model: row.model, + }, + ); } if entries.is_empty() { return Err(Error::new("", Repr::Empty)); @@ -190,6 +209,21 @@ impl Catalog { } } } + for (alias, target) in pointer_rows { + if !entries.contains_key(&target) { + problems.push(Error::from_key(&alias, Repr::AliasTargetMissing { target })); + } else if let Some(taken_by) = aliases.get(&alias) { + problems.push(Error::from_key( + &alias, + Repr::AliasTaken { + alias: alias.clone(), + taken_by: taken_by.clone(), + }, + )); + } else { + aliases.insert(alias, target); + } + } let fallback_generalizations = sections .iter() @@ -242,8 +276,8 @@ impl Catalog { } pub fn row_errors(model: &str, fields: &Fields) -> Vec { - let (_, _, errors) = read_row(fields); - errors + read_row(fields) + .problems .into_iter() .map(|error| error.prepend(model)) .collect() @@ -255,7 +289,13 @@ impl Catalog { (0..self.rows().entries.len() + self.sections.len()).map_while(move |index| match sections .next_if(|section| section.index == index) { - Some(section) => Some((section.key.as_str(), Written::Section(§ion.value))), + Some(section) => Some(( + section.key.as_str(), + match §ion.value { + Value::Object(fields) if !is_section(§ion.key) => Written::Entry(fields), + value => Written::Section(value), + }, + )), None => entries .next() .map(|(name, entry)| (name.as_str(), Written::Entry(&entry.fields))), @@ -406,16 +446,41 @@ impl Catalog { } } -fn read_row(fields: &Fields) -> (Option, Option>, Vec) { +struct RowRead { + model: Option, + aliases: Option>, + alias_of: Option, + problems: Vec, +} + +fn read_row(fields: &Fields) -> RowRead { let wire = EntryWire::read(fields); + if let Some(target) = wire.attributes.alias_of.clone() { + let other_fields = fields + .keys() + .filter(|key| *key != "alias_of") + .map(|key| Error::from_key(key, Repr::AliasRowField)); + return RowRead { + model: None, + aliases: None, + alias_of: Some(target), + problems: wire.problems.into_iter().chain(other_fields).collect(), + }; + } let aliases = wire.attributes.aliases.clone(); let timezone_error = unknown_timezone(&wire); - match Model::try_from(wire) { - Ok(model) => (Some(model), aliases, timezone_error.into_iter().collect()), + let (model, problems) = match Model::try_from(wire) { + Ok(model) => (Some(model), timezone_error.into_iter().collect()), Err(mut errors) => { errors.extend(timezone_error); - (None, aliases, errors) + (None, errors) } + }; + RowRead { + model, + aliases, + alias_of: None, + problems, } } diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs index 89e27c05f46..846398f7155 100644 --- a/litellm-rust/crates/model-catalog/src/error.rs +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -67,6 +67,12 @@ pub(crate) enum Repr { #[error("alias {alias:?} already belongs to {taken_by}")] #[strum_discriminants(strum(serialize = "alias_taken"))] AliasTaken { alias: String, taken_by: String }, + #[error("alias_of {target:?} is not a model name")] + #[strum_discriminants(strum(serialize = "alias_target_missing"))] + AliasTargetMissing { target: String }, + #[error("an alias_of row carries no other field")] + #[strum_discriminants(strum(serialize = "alias_row_field"))] + AliasRowField, #[error("{first} and {second} price the same meter under the same conditions")] #[strum_discriminants(strum(serialize = "duplicate_price"))] DuplicatePrice { first: String, second: String }, @@ -141,6 +147,8 @@ impl Error { | Repr::UnknownTimezone { .. } | Repr::AliasShadowsModel { .. } | Repr::AliasTaken { .. } + | Repr::AliasTargetMissing { .. } + | Repr::AliasRowField | Repr::DuplicatePrice { .. } | Repr::SuffixInsideBlock { .. } | Repr::TierRange @@ -327,6 +335,8 @@ mod tests { #[case::unknown_timezone(Repr::UnknownTimezone { zone: "invalid".into() }, Rejects::Row)] #[case::alias_shadows_model(Repr::AliasShadowsModel { alias: "model".into() }, Rejects::Row)] #[case::alias_taken(Repr::AliasTaken { alias: "alias".into(), taken_by: "owner".into() }, Rejects::Row)] + #[case::alias_target_missing(Repr::AliasTargetMissing { target: "owner".into() }, Rejects::Row)] + #[case::alias_row_field(Repr::AliasRowField, Rejects::Row)] #[case::duplicate_price(Repr::DuplicatePrice { first: "a".into(), second: "b".into() }, Rejects::Row)] #[case::suffix_inside_block(Repr::SuffixInsideBlock { block: "tiered_pricing" }, Rejects::Row)] #[case::tier_range(Repr::TierRange, Rejects::Row)] diff --git a/litellm-rust/crates/model-catalog/src/schema.rs b/litellm-rust/crates/model-catalog/src/schema.rs index 6fba985a8bd..60c755fe94f 100644 --- a/litellm-rust/crates/model-catalog/src/schema.rs +++ b/litellm-rust/crates/model-catalog/src/schema.rs @@ -104,9 +104,18 @@ pub fn registry_json_schema() -> Schema { "title".into(), "LiteLLM model prices and context window registry".into(), ); + definitions.insert( + "aliasRow".into(), + json!({ + "type": "object", + "required": ["alias_of"], + "properties": {"alias_of": {"type": "string"}}, + "additionalProperties": false, + }), + ); root.insert( "additionalProperties".into(), - json!({"$ref": "#/$defs/modelEntry"}), + json!({"anyOf": [{"$ref": "#/$defs/modelEntry"}, {"$ref": "#/$defs/aliasRow"}]}), ); root.insert("$defs".into(), Value::Object(definitions)); Schema::from(root.clone()) @@ -153,6 +162,7 @@ fn decorate_model_entry(schema: &mut Value) { let rates = json!({FLAT_KEY_PATTERN: {"type": "number", "minimum": 0}}); let object = schema.as_object_mut().unwrap(); object.insert("required".into(), json!(["litellm_provider"])); + object.insert("not".into(), json!({"required": ["alias_of"]})); object.insert("additionalProperties".into(), Value::Bool(true)); object.insert( "patternProperties".into(), diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 401634f8c62..52cd2e8525f 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -47,6 +47,8 @@ macro_rules! attributes { } attributes! { + /// Model this row is another name for; such a row carries no other key. + alias_of: String, /// Other names this entry is served under. aliases: Vec, audio_transcription_config: String, diff --git a/litellm-rust/crates/model-catalog/tests/catalog.rs b/litellm-rust/crates/model-catalog/tests/catalog.rs index 31f907689d3..679c67412c8 100644 --- a/litellm-rust/crates/model-catalog/tests/catalog.rs +++ b/litellm-rust/crates/model-catalog/tests/catalog.rs @@ -139,6 +139,18 @@ fn a_bad_entry_is_reported_by_model_and_key_and_kept_raw() { json!({"First": {"litellm_provider": "t", "aliases": ["shared"]}, "Second": {"litellm_provider": "t", "aliases": ["shared"]}}), r#"Second: alias "shared" already belongs to First"# )] +#[case::alias_of_target_missing( + json!({"Real": {"litellm_provider": "t"}, "ghost-latest": {"alias_of": "Ghost"}}), + r#"ghost-latest: alias_of "Ghost" is not a model name"# +)] +#[case::alias_of_row_with_other_fields( + json!({"Real": {"litellm_provider": "t"}, "real-latest": {"alias_of": "Real", "mode": "chat"}}), + "real-latest.mode: an alias_of row carries no other field" +)] +#[case::alias_of_claimed_by_a_list( + json!({"Real": {"litellm_provider": "t"}, "real-latest": {"alias_of": "Real"}, "Other": {"litellm_provider": "t", "aliases": ["real-latest"]}}), + r#"real-latest: alias "real-latest" already belongs to Other"# +)] #[case::aliases_not_a_list( json!({"a": {"litellm_provider": "t", "aliases": "bad"}}), "a.aliases: invalid value: invalid type: string \"bad\", expected a sequence" @@ -399,3 +411,70 @@ fn a_bad_value_keeps_its_json_error_as_the_source_and_names_its_key_and_field() assert_eq!(error.key().as_deref(), Some("open/router")); assert_eq!(error.field().as_deref(), Some("input_cost_per_token")); } + +#[rstest] +fn an_alias_of_row_is_served_as_its_target_and_written_as_itself() { + let catalog = catalog(json!({ + "Real": {"litellm_provider": "t", "input_cost_per_token": 1e-6}, + "real-latest": {"alias_of": "Real"}, + "After": {"litellm_provider": "t"} + })); + let found = catalog.lookup("REAL-LATEST").unwrap(); + assert_eq!( + (found.matched_key, found.canonical_key), + ("real-latest", "Real") + ); + assert_eq!( + found.entry.field("input_cost_per_token"), + Some(&json!(1e-6)) + ); + assert_eq!(catalog.model_count(), 2); + assert_eq!( + catalog.served().map(|(key, _)| key).collect::>(), + ["Real", "After", "real-latest"] + ); + assert_eq!( + written(&catalog), + r#"{"Real":{"litellm_provider":"t","input_cost_per_token":1e-6},"real-latest":{"alias_of":"Real"},"After":{"litellm_provider":"t"}}"# + ); +} + +#[rstest] +fn a_dangling_alias_of_row_is_rejected_and_dropped_from_the_written_document() { + let parsed = loaded(json!({ + "Real": {"litellm_provider": "t"}, + "ghost-latest": {"alias_of": "Ghost"} + })); + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(removed.len(), 1); + assert!(catalog.lookup("ghost-latest").is_none()); + assert_eq!(written(&catalog), r#"{"Real":{"litellm_provider":"t"}}"#); +} + +#[rstest] +fn an_alias_of_row_with_other_fields_is_never_served() { + let parsed = loaded(json!({ + "Real": {"litellm_provider": "t"}, + "real-latest": {"alias_of": "Real", "mode": "chat"} + })); + assert!(parsed.value.lookup("real-latest").is_none()); + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(removed.len(), 1); + assert!(catalog.lookup("real-latest").is_none()); + assert_eq!(written(&catalog), r#"{"Real":{"litellm_provider":"t"}}"#); +} + +#[rstest] +fn a_pointer_row_that_repeats_a_listed_alias_is_dropped_and_the_model_is_kept() { + let parsed = loaded(json!({ + "Real": {"litellm_provider": "t", "aliases": ["real-latest"]}, + "real-latest": {"alias_of": "Real"} + })); + let (catalog, removed) = parsed.value.without_rejected(&parsed.errors); + assert_eq!(removed.len(), 1); + assert_eq!(catalog.lookup("real-latest").unwrap().canonical_key, "Real"); + assert_eq!( + written(&catalog), + r#"{"Real":{"litellm_provider":"t","aliases":["real-latest"]}}"# + ); +} diff --git a/litellm-rust/crates/model-catalog/tests/schema.rs b/litellm-rust/crates/model-catalog/tests/schema.rs index 464cf069cac..5fe5832cb58 100644 --- a/litellm-rust/crates/model-catalog/tests/schema.rs +++ b/litellm-rust/crates/model-catalog/tests/schema.rs @@ -35,6 +35,7 @@ fn generated_registry_schema_validates_checked_in_catalog() { #[case(json!({"example": {"litellm_provider": "test"}}))] #[case(json!({"example": {"litellm_provider": "test", "future_field": true}}))] #[case(json!({"sample_spec": {"litellm_provider": "placeholder"}}))] +#[case(json!({"example": {"litellm_provider": "test"}, "example-latest": {"alias_of": "example"}}))] fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value) { assert!(registry_validator().is_valid(&document)); } @@ -54,6 +55,9 @@ fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value) #[case::empty_windows(json!({"litellm_provider": "test", "off_peak_pricing": {"windows": []}}))] #[case::invalid_weekday(json!({"litellm_provider": "test", "off_peak_pricing": {"windows": [{"hours_utc": "00:00-01:00", "weekdays": [0]}]}}))] #[case::invalid_aliases(json!({"litellm_provider": "test", "aliases": "wrong"}))] +#[case::alias_of_on_a_model(json!({"litellm_provider": "test", "alias_of": "other"}))] +#[case::alias_row_with_other_fields(json!({"alias_of": "other", "mode": "chat"}))] +#[case::alias_of_not_a_string(json!({"alias_of": 4}))] #[case::non_object_model(json!(4))] fn generated_registry_schema_rejects_invalid_entries(#[case] entry: Value) { assert!(!registry_validator().is_valid(&json!({"example": entry}))); diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index e452befe295..d2045c947fe 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -580,21 +580,25 @@ def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, .. def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]: """ - Expand ``aliases`` lists in model cost entries into top-level entries. + Expand alias rows into top-level entries that share their owner's ``AliasedCostEntry``. - Each alias gets a reference to the **same** ``AliasedCostEntry`` as the - canonical entry, which carries every name it is served under. The - ``aliases`` key is removed from the entry so downstream code never sees it. + A row whose ``alias_of`` names another row is served as that row, and an + ``aliases`` list on a row adds further names for it. Every name in a group + references the **same** ``AliasedCostEntry``, which carries every name it is + served under, and neither key survives into the served entries. - If an alias collides with an existing canonical entry the alias is - skipped and a warning is logged. + An alias that collides with a canonical entry or another alias, or a row whose + ``alias_of`` names nothing, is skipped and a warning is logged. """ + pointer_rows: Final = { + name: target for name, info in model_cost.items() if isinstance(target := info.get("alias_of"), str) + } alias_owners: Final[dict[str, str]] = {} keys_with_aliases: Final[list[str]] = [] for model_name, model_info in model_cost.items(): aliases: object = model_info.get("aliases") - if aliases is None: + if aliases is None or model_name in pointer_rows: continue keys_with_aliases.append(model_name) if not isinstance(aliases, list): @@ -607,7 +611,7 @@ def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, if not aliases: continue for alias in aliases: - if alias in model_cost: + if alias in model_cost and alias not in pointer_rows: verbose_logger.warning( "LiteLLM model alias conflict: alias '%s' (from '%s') " "already exists as a canonical entry — skipping.", @@ -625,9 +629,26 @@ def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, continue alias_owners[alias] = model_name + for alias, target in pointer_rows.items(): + if target not in model_cost or target in pointer_rows: + verbose_logger.warning( + "LiteLLM model alias '%s' points at '%s', which is not a model entry, dropping it.", alias, target + ) + elif alias in alias_owners: + verbose_logger.warning( + "LiteLLM model alias conflict: alias '%s' (pointing at '%s') " + "was already claimed by another entry — skipping.", + alias, + target, + ) + else: + alias_owners[alias] = target + # Remove the ``aliases`` key from entries so it doesn't pollute model info for key in keys_with_aliases: model_cost[key].pop("aliases", None) + for alias in pointer_rows.keys() - alias_owners.keys(): + model_cost.pop(alias) for owner, names in _alias_groups(alias_owners).items(): model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names))) diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index d44169ae28a..6267b841a95 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -130,6 +130,30 @@ def test_finalize_pops_key_and_installs_rules(): set_fallback_generalizations(previous) +def test_alias_of_rows_are_served_as_their_target_and_dangling_ones_are_dropped( + caplog: pytest.LogCaptureFixture, +) -> None: + raw: Final = { + "real": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6}, + "other": {"litellm_provider": "openai", "mode": "chat", "aliases": ["twin-latest"]}, + "real-latest": {"alias_of": "real"}, + "twin-latest": {"alias_of": "real"}, + "ghost-latest": {"alias_of": "ghost"}, + "chain-latest": {"alias_of": "real-latest"}, + "listed-latest": {"alias_of": "missing", "aliases": ["extra"]}, + } + with caplog.at_level("WARNING"): + finalized: Final = _finalize_model_cost_map(raw) + assert finalized["real-latest"] == finalized["real"] + assert "alias_of" not in finalized["real"] + assert set(finalized) == {"real", "other", "real-latest", "twin-latest"} + assert finalized["real-latest"]["litellm_provider"] == "openai" + assert finalized["twin-latest"] == finalized["other"] + assert {"ghost-latest", "chain-latest", "listed-latest", "twin-latest"} == { + record.getMessage().split("'")[1] for record in caplog.records + } + + def test_finalize_with_no_block_clears_rules(): previous = list(get_fallback_generalization_rules()) try: @@ -806,6 +830,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -815,6 +840,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -848,6 +874,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -862,6 +889,7 @@ _SCALAR_DEFAULTS = { "cohere_key": getattr(litellm, "cohere_key", None), } + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestCheckIsValidDict: """Unit tests for _check_is_valid_dict.""" @@ -886,6 +914,7 @@ class TestCheckIsValidDict: """Non-empty dict should pass.""" assert GetModelCostMap._check_is_valid_dict({"model": {}}) is True + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestCheckModelCountNotReduced: """Unit tests for _check_model_count_not_reduced.""" @@ -940,6 +969,7 @@ class TestCheckModelCountNotReduced: is True ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestValidateModelCostMap: """Unit tests for validate_model_cost_map (combines both checks).""" @@ -976,6 +1006,7 @@ class TestValidateModelCostMap: is True ) + def _fetch_with_single_outcome(monkeypatch, outcome): from litellm.litellm_core_utils import get_model_cost_map as module @@ -1051,6 +1082,7 @@ class TestGetModelCostMapFallback: assert isinstance(result, dict) assert len(result) > 0 + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBackupModelCostMapExists: """Validates the local backup file is always present and valid.""" @@ -1066,6 +1098,7 @@ class TestBackupModelCostMapExists: backup = GetModelCostMap.load_local_model_cost_map() assert len(backup) > 100, f"Backup has only {len(backup)} models, expected > 100" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBadHostedModelCostMap: """ @@ -1130,12 +1163,14 @@ class TestBadHostedModelCostMap: finally: litellm.model_cost = original + @pytest.fixture() def _vcr_outcome_gate_local_testing(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -1188,6 +1223,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS_local_testing = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -1208,6 +1244,7 @@ _SCALAR_DEFAULTS_local_testing = { "api_key": getattr(litellm, "api_key", None), } + @pytest.fixture(scope="module") def setup_and_teardown_local_testing(): """ @@ -1230,6 +1267,7 @@ def setup_and_teardown_local_testing(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", @@ -1241,6 +1279,7 @@ def test_get_model_cost_map(): except Exception as e: pytest.fail(f"An exception occurred: {e}") + @pytest.mark.usefixtures( "_vcr_outcome_gate_local_testing", "isolate_litellm_state", diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 59520938667..44bdd187da4 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1029,6 +1029,8 @@ _COST_MAP_ROW_SCHEMA: Final = { "audio_transcription_config": {"type": "string"}, "deprecation_date": {"type": "string"}, "successor_model": {"type": "string"}, + "aliases": {"type": "array", "items": {"type": "string"}}, + "alias_of": {"type": "string"}, "deprecation_source": {"type": "string"}, "input_cost_per_audio_per_second": {"type": "number"}, "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, From 268bc0345c53409342a6da56a96134034040aabe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gau=C2=B7Beo?= Date: Sun, 8 Nov 2026 01:35:18 +0700 Subject: [PATCH 06/17] feat(cost-map): add moonshot/kimi-k3.1-pro-highspeed pricing (#50521) (#50538) --- ...odel_prices_and_context_window_backup.json | 22 +++++++++++++++++++ model_prices_and_context_window.json | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3346fb66ecc..ac5e99c9313 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -44489,6 +44489,28 @@ "supports_response_schema": true, "supports_vision": true }, + "moonshot/kimi-k3.1-pro-highspeed": { + "cache_creation_input_token_cost": 0.000004, + "cache_creation_input_token_cost_above_1hr": 0.000008, + "cache_read_input_token_cost": 2e-7, + "input_cost_per_token": 0.000004, + "litellm_provider": "moonshot", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 0.00002, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://platform.kimi.ai/docs/pricing/chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, "moonshot/kimi-k3": { "cache_creation_input_token_cost": 3e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3346fb66ecc..ac5e99c9313 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -44489,6 +44489,28 @@ "supports_response_schema": true, "supports_vision": true }, + "moonshot/kimi-k3.1-pro-highspeed": { + "cache_creation_input_token_cost": 0.000004, + "cache_creation_input_token_cost_above_1hr": 0.000008, + "cache_read_input_token_cost": 2e-7, + "input_cost_per_token": 0.000004, + "litellm_provider": "moonshot", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 0.00002, + "reasoning_effort_levels": [ + "low", + "high", + "max" + ], + "source": "https://platform.kimi.ai/docs/pricing/chat", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, "moonshot/kimi-k3": { "cache_creation_input_token_cost": 3e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, From cdbb4879263091a4cba2943519ae158adf7f8668 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:56:30 -0800 Subject: [PATCH 07/17] refactor: read lazy attributes with getattr instead of a TypedDict wrapper (#50543) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/_lazy_imports.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 1ea4bb52668..2177edb4ca8 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -6,8 +6,6 @@ from collections.abc import Callable, Mapping from types import MappingProxyType, ModuleType from typing import TYPE_CHECKING, Final -from typing_extensions import ReadOnly, TypedDict - from ._lazy_imports_registry import LITELLM_ATTRIBUTES, UTILS_ATTRIBUTES if TYPE_CHECKING: @@ -61,22 +59,14 @@ def _cache(namespace: str, name: str, value: object) -> object: return value -class _AttributeView(TypedDict): - value: ReadOnly[object] - - -def _module_attribute(module: ModuleType, attr_name: str) -> object: - attribute: Final[_AttributeView] = {"value": getattr(module, attr_name)} - return attribute["value"] - - def _import(namespace: str, name: str, location: tuple[str, str]) -> object: cached: Final = _cached(namespace, name) if cached is not _NOT_LOADED: return cached module_path, attr_name = location module: Final[ModuleType] = importlib.import_module(module_path, package="litellm") - return _cache(namespace, name, _module_attribute(module, attr_name)) + attribute: Final[object] = getattr(module, attr_name) + return _cache(namespace, name, attribute) def _instance_of(class_name: str) -> Callable[[], object]: From d0c16008c8e80b992f9bdb90078e17ee59cda5b4 Mon Sep 17 00:00:00 2001 From: DDT <1786035110@qq.com> Date: Sun, 8 Nov 2026 03:32:59 +0800 Subject: [PATCH 08/17] fix(chatgpt): round-trip encrypted reasoning through /v1/messages (#49076) * fix(chatgpt): round-trip encrypted reasoning through /v1/messages The chat bridge stored encrypted reasoning on reasoning_items, the Anthropic adapter never read that field, stream chunks that only carried thinking blocks were dropped, and the messages adapter stripped the signed blocks before a chatgpt request could turn them back into reasoning items. Co-authored-by: DDT <1786035110@qq.com> * fix(anthropic): keep Responses reasoning on tool turns and retry once when a backend cannot decrypt it * fix(anthropic): stop logging rejected encrypted reasoning Quote the retry helper return type so importing litellm no longer evaluates a string alias with |. The warning keeps provider, model, status, and error code, and leaves the ciphertext out. * fix(anthropic): keep encrypted reasoning on OpenAI thinking and xAI web search turns, and stream signatures only to Messages clients The keep check now knows when the handler moves an OpenAI thinking turn to Responses, and when web_search_options sends an xAI turn there. The Responses-to-chat stream iterator adds the signed reasoning chunk only when the caller is /v1/messages, so /v1/chat/completions streams stay unchanged. The reasoning retry fires only when the translated request still carries encrypted reasoning. * fix(anthropic): keep each streamed reasoning summary with its own item, and read the Messages marker before the bridge call * fix(anthropic): send openai/ Messages turns again once without encrypted reasoning a backend rejects The openai/ Responses path on /v1/messages had no retry, so a chatgpt/ block left in history answered 400 on every later openai/ turn. That path now sends the turn once more without encrypted reasoning, the same way the chat bridge does. The retry matches only invalid_encrypted_content or a "could not be verified" message naming the content, so the exception mapper's own hint no longer triggers it. The xAI web search keep path is backed out, since xAI sends no encrypted reasoning over the bridge. * fix(anthropic): keep every non-streamed reasoning item for Messages clients and read a deployment's reasoning_summary in the keep check * fix(anthropic): log only the outcome of a turn sent again without rejected reasoning A /v1/messages turn whose replayed reasoning the backend rejects is sent once more without it. The rejected first attempt ran the failure callbacks on the shared logging object, which logged a failure for a turn that then succeeded, kept the retry's own failure from being logged, and freed the parallel-request slot while the retry was still in flight. The first attempt now runs inside resent_on, and the client wrapper skips failure logging and the budget unbind for a failure that matches it. The retry runs outside that context, so its own outcome logs as usual. * test(anthropic): type the parameters of the failure-recording test loggers * fix(anthropic): skip the SDK num_retries retry for an attempt that is sent again * fix(anthropic): stop SDK retries from resending a rejected block or clearing the global num_retries A transient 429 or 500 on a /v1/messages turn that carries a replayed encrypted block was retried by an SDK call's num_retries loop, which sent the block again until every retry was rejected. On async calls the loop then raised the first transient error, so the turn failed instead of being sent again without the block. The retry loop now stops at that rejection and passes it on, so the turn is sent again once without encrypted blocks. A rejected first try on a sync call also cleared the process-wide litellm.num_retries, so later calls in the process were no longer retried. That setting now stays as configured. * fix(anthropic): stop a cancelled SDK retry from starting another call The resent-failure retry predicate returned true for BaseException subclasses such as asyncio.CancelledError, so a cancelled acompletion_with_retries or aresponses_with_retries call could start another provider call. Retry only Exception subclasses again, as tenacity's default did, while still skipping a resent failure. * test(anthropic): type the retry helper parameter in the cancellation test --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/_internal_context.py | 19 + .../handler.py | 8 + .../transformation.py | 108 ++- .../prompt_templates/reasoning_utils.py | 95 +++ .../litellm_core_utils/streaming_handler.py | 1 + .../anthropic/messages/adapters/handler.py | 230 ++++-- .../messages/adapters/streaming_iterator.py | 27 +- .../messages/adapters/transformation.py | 175 +++-- .../messages/responses_adapters/handler.py | 168 +++- litellm/main.py | 33 +- litellm/responses/bridge_check.py | 41 +- litellm/utils.py | 34 +- ...responses_transformation_transformation.py | 380 ++++++++++ .../test_streaming_handler.py | 31 + ...hropic_messages_adapters_transformation.py | 717 +++++++++++++++++- .../test_responses_adapters_handler.py | 292 +++++++ tests/unit/test_main.py | 15 +- 17 files changed, 2191 insertions(+), 183 deletions(-) diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 416563d5093..e97d79a95bc 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -44,6 +44,8 @@ _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", de _mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False) +_resent_on: Final[ContextVar[Callable[[Exception], bool] | None]] = ContextVar("resent_on", default=None) + _request_parent_otel_span: Final[ContextVar[Span | None]] = ContextVar("request_parent_otel_span", default=None) @@ -168,6 +170,23 @@ def in_mirrored_call() -> bool: return _mirrored_call.get() +@contextmanager +def resent_on(rejection: Callable[[Exception], bool]) -> Generator[None]: + """An attempt its caller sends again when it fails with ``rejection``. That failure is not the request's + outcome, so it is neither logged as the request's failure nor retried, and its budget reservation and rate-limit + slot stay with the attempt sent again.""" + token: Final = _resent_on.set(rejection) + try: + yield + finally: + _restore(_resent_on, token) + + +def is_resent_failure(error: Exception) -> bool: + rejection: Final = _resent_on.get() + return rejection is not None and rejection(error) + + def set_request_parent_otel_span(span: Span) -> None: """Anchor the proxy request's OTEL span for datastore service spans that no call site passes a parent to. Never reset: it dies with the request task and rides the tasks it spawns, like ``request.state``.""" diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index f6879c313de..861ffd91bfa 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -182,6 +182,9 @@ class ResponsesToCompletionBridgeHandler: model_response: Final = validated_kwargs["model_response"] logging_obj: Final = validated_kwargs["logging_obj"] custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] + from .transformation import serves_anthropic_messages + + streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} logging_obj.record_bridged_chat_request(optional_params, litellm_params) @@ -262,6 +265,7 @@ class ResponsesToCompletionBridgeHandler: cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), replays_openai_history=custom_llm_provider == "openai", + streams_reasoning_signatures=streams_reasoning_signatures, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -284,6 +288,9 @@ class ResponsesToCompletionBridgeHandler: model_response: Final = validated_kwargs["model_response"] logging_obj: Final = validated_kwargs["logging_obj"] custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] + from .transformation import serves_anthropic_messages + + streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} logging_obj.record_bridged_chat_request(optional_params, litellm_params) @@ -367,6 +374,7 @@ class ResponsesToCompletionBridgeHandler: cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), replays_openai_history=custom_llm_provider == "openai", + streams_reasoning_signatures=streams_reasoning_signatures, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index fc7a6048538..a2de41a0c45 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -39,7 +39,12 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( parse_chat_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, iter_items +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + attribute_or_none, + drops_unsupported_params, + iter_items, +) from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( prompt_cache_breakpoint_with_readable_mode, @@ -48,6 +53,7 @@ from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils i ) from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( responses_reasoning_items_from_thinking_blocks, + thinking_blocks_from_reasoning_items, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( @@ -69,6 +75,8 @@ from litellm.responses.utils import ( from litellm.types.llms.openai import ( ChatCompletionAnnotation, ChatCompletionReasoningItem, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, @@ -77,7 +85,7 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import GenericStreamingChunk, ModelResponseStream +from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices if TYPE_CHECKING: from openai.types.responses import ResponseInputImageParam, ResponseOutputItem @@ -93,8 +101,6 @@ if TYPE_CHECKING: AllMessageValues, ChatCompletionFileObject, ChatCompletionImageObject, - ChatCompletionRedactedThinkingBlock, - ChatCompletionThinkingBlock, ChatCompletionToolReferenceObject, OpenAIMessageContentListBlock, ) @@ -264,6 +270,10 @@ class _BuiltReasoningItem(TypedDict): summary: Sequence[_ReasoningSummaryText] +def serves_anthropic_messages(logging_obj: object) -> bool: + return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value + + def _get_reasoning_items( msg: "AllMessageValues", ) -> list[ChatCompletionReasoningItem]: @@ -496,6 +506,77 @@ def _as_chat_reasoning_items( return cast(list[ChatCompletionReasoningItem], list(reasoning_items)) +def _with_stream_block_index( + block: Mapping[str, object], + index: int, + summary_streamed: bool, +) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock: + if block.get("type") == "redacted_thinking": + data: Final = block.get("data") + return ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=data if isinstance(data, str) else "", + index=index, + ) + thinking: Final = block.get("thinking") + signature: Final = block.get("signature") + return ChatCompletionThinkingBlock( + type="thinking", + thinking=thinking if isinstance(thinking, str) and not summary_streamed else "", + signature=signature if isinstance(signature, str) else None, + index=index, + ) + + +def _stream_output_index(chunk: Mapping[str, object]) -> int | None: + output_index: Final = chunk.get("output_index") + return output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else None + + +def _reasoning_done_thinking_blocks( + item: Mapping[str, object], + output_index: int | None, + summary_streamed: bool, +) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]: + index: Final = output_index if output_index is not None else 0 + return tuple( + _with_stream_block_index(block, index, summary_streamed) + for block in thinking_blocks_from_reasoning_items((item,)) + ) + + +def _reasoning_done_stream_chunk( + chunk: Mapping[str, object], + summary_streamed_outputs: frozenset[int | None], +) -> ModelResponseStream | None: + event_type: Final = chunk.get("type") + event_name: Final = ( + event_type.value + if isinstance(event_type, ResponsesAPIStreamEvents) + else event_type + if isinstance(event_type, str) + else None + ) + if event_name != ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: + return None + item: Final = as_str_mapping(chunk.get("item")) + if item is None or item.get("type") != "reasoning": + return None + output_index: Final = _stream_output_index(chunk) + signed_blocks: Final = _reasoning_done_thinking_blocks(item, output_index, output_index in summary_streamed_outputs) + if not signed_blocks: + return None + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(thinking_blocks=list(signed_blocks)), + finish_reason=None, + ) + ] + ) + + def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation: shifted: Final = { key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value @@ -1280,6 +1361,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): structured_output: bool = False, replay_openai_history: bool = False, completed: bool = False, + keeps_every_reasoning_item: bool = False, ) -> list["Choices"]: """ Convert Responses API output items to chat completion choices. @@ -1291,6 +1373,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): first message item becomes content replay_openai_history: Record the output's item sequence for the next turn completed: The response finished, so its item sequence is whole + keeps_every_reasoning_item: Keep each reasoning item before an answer, not only the last Returns: List of Choices objects @@ -1323,7 +1406,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for item in output_items: if isinstance(item, ResponseReasoningItem): pending_reasoning_items = ( - *(pending_reasoning_items if replay_openai_history else ()), + *(pending_reasoning_items if replay_openai_history or keeps_every_reasoning_item else ()), _build_reasoning_item( item_id=item.id, encrypted_content=getattr(item, "encrypted_content", None), @@ -1573,6 +1656,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) == "openai", completed=raw_response.status == "completed", + keeps_every_reasoning_item=serves_anthropic_messages(logging_obj), ) response_is_incomplete: Final = raw_response.status == "incomplete" or ( @@ -1637,6 +1721,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): json_mode: bool | None = False, structured_output: bool = False, replays_openai_history: bool = False, + streams_reasoning_signatures: bool = False, ) -> BaseModelResponseIterator: return OpenAiResponsesToChatCompletionStreamIterator( streaming_response, @@ -1644,6 +1729,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): json_mode, structured_output=structured_output, replays_openai_history=replays_openai_history, + streams_reasoning_signatures=streams_reasoning_signatures, ) def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]: @@ -2037,15 +2123,18 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): json_mode: bool | None = False, structured_output: bool = False, replays_openai_history: bool = False, + streams_reasoning_signatures: bool = False, ): super().__init__(streaming_response, sync_stream, json_mode) self._replays_openai_history: Final = replays_openai_history + self._streams_reasoning_signatures: Final = streams_reasoning_signatures self._chat_completion_id: str | None = None self._served_service_tier: str | None = None self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state self._structured_output: Final = structured_output self._first_message_item_done: bool = False self._reasoning_summary_streamed: bool = False + self._summary_streamed_outputs: frozenset[int | None] = frozenset() def _handle_string_chunk( self, str_line: Union[str, "BaseModel"] @@ -2263,7 +2352,6 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) ] ) - elif event_type == "response.output_text.delta": # Content part added to output content_part = parsed_chunk.get("delta", None) @@ -2380,9 +2468,16 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): """ verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk) self._remember_served_service_tier(chunk) + reasoning_chunk: Final = ( + _reasoning_done_stream_chunk(chunk, self._summary_streamed_outputs) + if self._streams_reasoning_signatures + else None + ) translated: Final = ( self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) + else reasoning_chunk + if reasoning_chunk is not None else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( chunk, tool_call_index_map=self._tool_call_index_map, @@ -2395,6 +2490,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): self._first_message_item_done = True if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"): self._reasoning_summary_streamed = True + self._summary_streamed_outputs = self._summary_streamed_outputs | {_stream_output_index(chunk)} return self._with_served_service_tier(self._with_stream_scoped_id(translated)) def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool: diff --git a/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py index 817b1fa5915..ff1f2e017e3 100644 --- a/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py @@ -5,6 +5,9 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from itertools import groupby from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move +from pydantic import TypeAdapter + +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.openai import ( ChatCompletionReasoningItem, ChatCompletionReasoningSummaryTextBlock, @@ -94,6 +97,90 @@ def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: return encrypted_content_from_signature(_encrypted_reasoning_field(block)) +_REASONING_SUMMARY_SEPARATOR: Final = "\n\n" + + +_OBJECT_TUPLE: Final = TypeAdapter(tuple[object, ...]) + + +def _object_sequence(value: object) -> tuple[object, ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + return () + return _OBJECT_TUPLE.validate_python(value) + + +def _iter_mappings(value: object) -> tuple[Mapping[str, object], ...]: + return tuple(mapping for item in _object_sequence(value) if (mapping := as_str_mapping(item)) is not None) + + +def _reasoning_summary_part_text(part: object) -> str: + mapping: Final = as_str_mapping(part) + if mapping is not None: + text: Final = mapping.get("text") + return text if isinstance(text, str) else "" + text_attr: Final = getattr(part, "text", None) + return text_attr if isinstance(text_attr, str) else "" + + +def _reasoning_item_summary_text(item: Mapping[str, object]) -> str: + parts: Final = tuple(_reasoning_summary_part_text(part) for part in _object_sequence(item.get("summary"))) + return _REASONING_SUMMARY_SEPARATOR.join(part for part in parts if part) + + +def thinking_block_from_reasoning_item( + item: object, +) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None: + """The chat thinking block that carries one Responses `reasoning` item back to an Anthropic client. + + `encrypted_content` rides the same `litellm_encrypted_reasoning:` signature the Responses + adapter uses. A summary becomes thinking text. Bytes with no summary become `redacted_thinking`, + because an empty thinking string is not something the client can replay. + """ + mapping: Final = as_str_mapping(item) + if mapping is None or mapping.get("type") not in (None, "reasoning"): + return None + encrypted_content: Final = mapping.get("encrypted_content") + if not isinstance(encrypted_content, str) or not encrypted_content: + return None + signature: Final = encrypted_reasoning_signature(encrypted_content) + text: Final = _reasoning_item_summary_text(mapping) + if not text: + return ChatCompletionRedactedThinkingBlock(type="redacted_thinking", data=signature) + return ChatCompletionThinkingBlock(type="thinking", thinking=text, signature=signature) + + +def thinking_blocks_from_reasoning_items( + items: object, +) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]: + return tuple( + block for item in _object_sequence(items) if (block := thinking_block_from_reasoning_item(item)) is not None + ) + + +def _block_encrypted_content(block: object) -> str | None: + mapping: Final = as_str_mapping(block) + if mapping is None: + return None + return encrypted_content_of_block(mapping) + + +def thinking_blocks_with_encrypted_reasoning( + thinking_blocks: object, + reasoning_items: object, +) -> tuple[Mapping[str, object], ...]: + """Thinking blocks already on the message, plus any encrypted reasoning item they do not already carry.""" + direct: Final = _iter_mappings(thinking_blocks) + represented: Final = frozenset( + content for block in direct if (content := _block_encrypted_content(block)) is not None + ) + extra: Final = tuple( + block + for block in thinking_blocks_from_reasoning_items(_iter_mappings(reasoning_items)) + if _block_encrypted_content(block) not in represented + ) + return (*extra, *direct) + + def is_encrypted_reasoning_block(block: object) -> bool: """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. @@ -195,6 +282,14 @@ def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: ) +def _content_carries_encrypted_reasoning(content: object) -> bool: + return any(is_encrypted_reasoning_block(block) for block in _object_sequence(content)) + + +def messages_carry_encrypted_reasoning(messages: Sequence[object]) -> bool: + return any(_content_carries_encrypted_reasoning(content) for content in anthropic_content_lists(messages)) + + def _strip_encrypted_reasoning_from_blocks( content: object, *, diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6839e11f8c7..f7bd86f3a0c 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1140,6 +1140,7 @@ class CustomStreamWrapper: and model_response.choices[0].delta.role is not None ) or (getattr(model_response.choices[0].delta, "reasoning_items", None) is not None) + or getattr(model_response.choices[0].delta, "thinking_blocks", None) ): return True else: diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 64c7ac62d03..70b539a14d4 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -1,5 +1,8 @@ -from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence +import re +from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass +from functools import partial from typing import ( TYPE_CHECKING, Final, @@ -11,17 +14,25 @@ from pydantic import TypeAdapter from typing_extensions import TypedDict import litellm +from litellm._internal_context import resent_on from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( + is_encrypted_reasoning_block, + thinking_blocks_with_encrypted_reasoning, +) from litellm.litellm_core_utils.tool_fields import ( ROUTES_WITHOUT_MCP_SERVERS, drops_mcp_servers, tools_without_mcp_servers, ) +from litellm.llms.anthropic.common_utils import strip_encrypted_reasoning_blocks_from_anthropic_messages from litellm.llms.anthropic.messages.adapters.transformation import ( AnthropicAdapter, + openai_thinking_routes_to_responses, ) from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, @@ -50,9 +61,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( ) from litellm.types.llms.openai import OpenAIWebSearchOptions from litellm.types.utils import ModelResponse -from litellm.utils import get_model_info if TYPE_CHECKING: + from litellm.chat_completions.dispatch import ChatResult from litellm.router import Router from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth @@ -70,22 +81,157 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object): custom_llm_provider: str +_ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]]) +_UNVERIFIED_ENCRYPTED_CONTENT: Final = re.compile(r"encrypted content \S+ could not be verified", re.IGNORECASE) +_PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]" + + @dataclass(frozen=True, slots=True) class _ChatTargetReasoning: effort: str summary: str | None = None +def _provider_error_code(body: object) -> str | None: + mapping: Final = as_str_mapping(body) + if mapping is None: + return None + nested: Final = as_str_mapping(mapping.get("error")) + source: Final = nested if nested is not None else mapping + code: Final = source.get("code") + return code if isinstance(code, str) else None + + +def _logged_rejection_code(error: litellm.BadRequestError) -> str: + if error.code == "invalid_encrypted_content" or _provider_error_code(error.body) == "invalid_encrypted_content": + return "invalid_encrypted_content" + if "invalid_encrypted_content" in str(error): + return "invalid_encrypted_content" + return "could_not_be_verified" + + +def _completion_text(completion_kwargs: _CompletionKwargs, key: str) -> str: + value: Final = completion_kwargs.get(key) + return value if isinstance(value, str) else "" + + +def _message_carries_encrypted_reasoning(message: object) -> bool: + mapping: Final = as_str_mapping(message) + if mapping is None: + return False + blocks: Final = thinking_blocks_with_encrypted_reasoning( + mapping.get("thinking_blocks"), mapping.get("reasoning_items") + ) + return any(is_encrypted_reasoning_block(block) for block in blocks) + + +def _sends_encrypted_reasoning(completion_kwargs: _CompletionKwargs) -> bool: + messages: Final = as_object_list(completion_kwargs.get("messages")) or () + return any(_message_carries_encrypted_reasoning(message) for message in messages) + + +def _rejects_encrypted_reasoning(error: litellm.BadRequestError) -> bool: + error_message: Final = str(error) + return ( + "invalid_encrypted_content" in error_message or _UNVERIFIED_ENCRYPTED_CONTENT.search(error_message) is not None + ) + + +def _is_rejected_reasoning(error: Exception) -> bool: + return isinstance(error, litellm.BadRequestError) and _rejects_encrypted_reasoning(error) + + +def resent_without_rejected_reasoning(sent_encrypted_reasoning: bool) -> AbstractContextManager[None]: + return resent_on(_is_rejected_reasoning) if sent_encrypted_reasoning else nullcontext() + + +def messages_without_rejected_reasoning( + messages: Sequence[Mapping[str, object]], + error: litellm.BadRequestError, + *, + sent_encrypted_reasoning: bool, + provider: str, + model: str, +) -> "_AnthropicMessages | None": + if not sent_encrypted_reasoning or not _rejects_encrypted_reasoning(error): + return None + verbose_logger.warning( + "Backend rejected replayed encrypted reasoning, retrying once without it. " + "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " + "provider=%s model=%s status=%s code=%s", + provider, + model, + error.status_code, + _logged_rejection_code(error), + ) + return _ANTHROPIC_MESSAGES.validate_python( + strip_encrypted_reasoning_blocks_from_anthropic_messages([dict(message) for message in messages]) + ) + + +def _messages_without_rejected_reasoning( + messages: _AnthropicMessages, + error: litellm.BadRequestError, + completion_kwargs: _CompletionKwargs, + sent_encrypted_reasoning: bool, +) -> "_AnthropicMessages | None": + return messages_without_rejected_reasoning( + messages, + error, + sent_encrypted_reasoning=sent_encrypted_reasoning, + provider=_completion_text(completion_kwargs, "custom_llm_provider"), + model=_completion_text(completion_kwargs, "model"), + ) + + +async def _acompletion_without_rejected_reasoning( + completion_kwargs: _CompletionKwargs, + messages: _AnthropicMessages, + prepare_replay: _PrepareReplay, +) -> tuple[_CompletionKwargs, "ChatResult"]: + sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(completion_kwargs) + try: + with resent_without_rejected_reasoning(sent_encrypted_reasoning): + return completion_kwargs, await litellm.acompletion(**completion_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning( + messages, error, completion_kwargs, sent_encrypted_reasoning + ) + if replayed_messages is None: + raise + replay_kwargs: Final = prepare_replay(replayed_messages) + return replay_kwargs, await litellm.acompletion(**replay_kwargs) + + +def _completion_without_rejected_reasoning( + completion_kwargs: _CompletionKwargs, + messages: _AnthropicMessages, + prepare_replay: _PrepareReplay, +) -> tuple[_CompletionKwargs, "ChatResult | Coroutine[object, object, ChatResult]"]: + sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(completion_kwargs) + try: + with resent_without_rejected_reasoning(sent_encrypted_reasoning): + return completion_kwargs, litellm.completion(**completion_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning( + messages, error, completion_kwargs, sent_encrypted_reasoning + ) + if replayed_messages is None: + raise + replay_kwargs: Final = prepare_replay(replayed_messages) + return replay_kwargs, litellm.completion(**replay_kwargs) + + +def _is_compaction_block(block: object) -> bool: + mapping: Final = as_str_mapping(block) + return mapping is not None and mapping.get("type") == "compaction" + + def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool: """Return True when any message carries a ``compaction`` content block.""" - for msg in messages: - content = msg.get("content") - if not isinstance(content, list): - continue - for block in content: - if isinstance(block, dict) and block.get("type") == "compaction": - return True - return False + return any( + any(map(_is_compaction_block, content)) for msg in messages if isinstance(content := msg.get("content"), list) + ) def _proxy_router_fallback() -> "Router | None": @@ -419,34 +565,17 @@ class LiteLLMMessagesToCompletionTransformationHandler: If the user provides a `summary` field in the thinking dict, it is passed through to the OpenAI reasoning params (opt-in per OpenAI spec). """ - custom_llm_provider = completion_kwargs.get("custom_llm_provider") - if custom_llm_provider is None: - try: - _, inferred_provider, _, _ = litellm.utils.get_llm_provider( - model=cast(str, completion_kwargs.get("model")) - ) - custom_llm_provider = inferred_provider - except Exception: - custom_llm_provider = None - - if custom_llm_provider != "openai": - return - - if not isinstance(thinking, dict) or thinking.get("type") != "enabled": - return - model: Final = completion_kwargs.get("model") - try: - model_info: Final = get_model_info(model=cast(str, model), custom_llm_provider=custom_llm_provider) - if model_info and model_info.get("supports_reasoning") is False: - # Model doesn't support reasoning/responses API, don't route - return - except Exception: - pass + custom_llm_provider: Final = completion_kwargs.get("custom_llm_provider") + if not isinstance(model, str) or thinking is None: + return + if not openai_thinking_routes_to_responses( + model, custom_llm_provider if isinstance(custom_llm_provider, str) else None, thinking + ): + return - if isinstance(model, str) and model and "responses/" not in model: - local_model: Final = model.removeprefix(f"{custom_llm_provider}/") - completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}" + if "responses/" not in model: + completion_kwargs["model"] = f"openai/responses/{model.removeprefix('openai/')}" auto_summary: Final = is_reasoning_auto_summary_enabled() @@ -677,6 +806,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: custom_llm_provider=target_provider, api_base=api_base if isinstance(api_base, str) else None, deployment_model_info=as_str_mapping(provider_kwargs.get("model_info")), + reasoning_summary=peek_reasoning_summary_aliases(provider_kwargs), drop_params=drop_params, ) @@ -804,12 +934,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: parent_trace_id=sub_call_trace_id, ) - ( - completion_kwargs, - tool_name_mapping, - ) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + prepare: Final = partial( + LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs, max_tokens=max_tokens, - messages=effective_messages, model=model, metadata=metadata, stop_sequences=stop_sequences, @@ -824,8 +951,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: output_format=output_format, extra_kwargs=kwargs, ) - - completion_response: Final = await litellm.acompletion(**completion_kwargs) + initial_kwargs, tool_name_mapping = prepare(messages=effective_messages) + completion_kwargs, completion_response = await _acompletion_without_rejected_reasoning( + initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0] + ) logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) if stream: @@ -951,12 +1080,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages effective_system: Final = polyfill_result.system if polyfill_result is not None else system - ( - completion_kwargs, - tool_name_mapping, - ) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + prepare: Final = partial( + LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs, max_tokens=max_tokens, - messages=effective_messages, model=model, metadata=metadata, stop_sequences=stop_sequences, @@ -971,8 +1097,10 @@ class LiteLLMMessagesToCompletionTransformationHandler: output_format=output_format, extra_kwargs=kwargs, ) - - completion_response: Final = litellm.completion(**completion_kwargs) + initial_kwargs, tool_name_mapping = prepare(messages=effective_messages) + completion_kwargs, completion_response = _completion_without_rejected_reasoning( + initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0] + ) logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs) if stream: diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index 87be996ebbf..cf96c1c5fda 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -132,14 +132,21 @@ def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str: assert_never(delta_type) -def _first_thinking_block_index(chunk: "ModelResponseStream") -> int | None: +def _first_thinking_block(chunk: "ModelResponseStream") -> Mapping[str, object] | None: delta: Final = _optional_attr(chunk.choices[0], "delta") if chunk.choices else None blocks: Final = _optional_attr_sequence(delta, "thinking_blocks") - first_block: Final = as_str_mapping(blocks[0]) if blocks else None - index: Final = first_block.get("index") if first_block is not None else None + return as_str_mapping(blocks[0]) if blocks else None + + +def _thinking_block_index(block: Mapping[str, object] | None) -> int | None: + index: Final = block.get("index") if block is not None else None return index if isinstance(index, int) else None +def _closes_thinking_block(block: Mapping[str, object] | None) -> bool: + return block is not None and _thinking_block_index(block) is not None and bool(block.get("signature")) + + def _optional_str_attr(obj: object, name: str) -> str | None: value: Final = _optional_attr(obj, name) return value if isinstance(value, str) else None @@ -632,6 +639,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): sent_content_block_finish: bool = False current_content_block_type: Literal["text", "tool_use", "thinking", "redacted_thinking"] = "text" current_thinking_block_index: int | None = None + current_thinking_block_closed: bool = False sent_last_message: bool = False holding_chunk: ContentBlockDelta | None = None holding_stop_reason_chunk: MessageBlockDelta | None = None @@ -1575,9 +1583,16 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): original_name: Final = self.tool_name_mapping.get(truncated_name, truncated_name) tool_block["name"] = original_name - thinking_block_index: Final = _first_thinking_block_index(chunk) + thinking_block: Final = _first_thinking_block(chunk) + thinking_block_index: Final = _thinking_block_index(thinking_block) previous_thinking_block_index: Final = self.current_thinking_block_index - if thinking_block_index is not None or block_type != self.current_content_block_type: + follows_closed_thinking_block: Final = self.current_thinking_block_closed and block_type == "thinking" + self.current_thinking_block_closed = _closes_thinking_block(thinking_block) + if ( + thinking_block_index is not None + or block_type != self.current_content_block_type + or follows_closed_thinking_block + ): self.current_thinking_block_index = thinking_block_index if block_type != self.current_content_block_type: @@ -1585,7 +1600,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): self.current_content_block_start = content_block_start return True - if ( + if follows_closed_thinking_block or ( thinking_block_index is not None and previous_thinking_block_index is not None and thinking_block_index != previous_thinking_block_index diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 5e4fba6d706..0c564a194dc 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -2,13 +2,14 @@ import copy import hashlib import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import JsonValue, TypeAdapter import litellm -from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items, normalize_drop_params from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.messages.utils import ( @@ -153,6 +154,7 @@ from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils i ) from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( reasoning_content_from_thinking_blocks, + thinking_blocks_with_encrypted_reasoning, ) from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, @@ -258,27 +260,72 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider) -def _is_azure_responses_target( +@dataclass(frozen=True, slots=True) +class ChatRouteInputs: + tools: Sequence[Mapping[str, object]] | None = None + reasoning_effort: str | Mapping[str, object] | None = None + api_base: str | None = None + thinking: Mapping[str, object] | None = None + reasoning_summary: object | None = None + + +_NO_ROUTE_INPUTS: Final = ChatRouteInputs() + + +def _inferred_provider(model: str) -> str | None: + try: + return litellm.utils.get_llm_provider(model=model)[1] + except Exception: + return None + + +def openai_thinking_routes_to_responses( + model: str | None, + custom_llm_provider: str | None, + thinking: Mapping[str, object] | None, +) -> bool: + if not model or thinking is None or thinking.get("type") != "enabled": + return False + provider: Final = custom_llm_provider if custom_llm_provider is not None else _inferred_provider(model) + if provider != "openai": + return False + try: + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider=provider) + except Exception: + return True + return model_info.get("supports_reasoning") is not False + + +def _keeps_encrypted_reasoning_for_responses( model: str | None, custom_llm_provider: str | None, deployment_model_info: Mapping[str, object] | None, + route: ChatRouteInputs, ) -> bool: - if not model: - return False - if custom_llm_provider not in (None, LlmProviders.AZURE.value): - return False - if custom_llm_provider is None and not model.startswith("azure/"): - return False + from litellm.responses.bridge_check import chat_completion_ends_in_responses_api - from litellm.responses.bridge_check import responses_api_bridge_check - - resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) - model_info, _ = responses_api_bridge_check( - model=resolved_model, - custom_llm_provider=provider, - deployment_model_info=deployment_model_info, + return openai_thinking_routes_to_responses( + model, custom_llm_provider, route.thinking + ) or chat_completion_ends_in_responses_api( + model, + custom_llm_provider, + deployment_model_info, + tools=route.tools, + reasoning_effort=route.reasoning_effort, + reasoning_summary=route.reasoning_summary, + api_base=route.api_base, ) - return model_info.get("mode") == "responses" + + +def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], ...]: + return thinking_blocks_with_encrypted_reasoning( + getattr(source, "thinking_blocks", None), + getattr(source, "reasoning_items", None), + ) + + +def _streamed_thinking_blocks(delta: object) -> tuple[Mapping[str, object], ...]: + return tuple(block for item in attribute_items(delta, "thinking_blocks") if (block := as_str_mapping(item))) def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam: @@ -317,6 +364,7 @@ class AnthropicAdapter: custom_llm_provider: str | None = None, api_base: str | None = None, deployment_model_info: Mapping[str, object] | None = None, + reasoning_summary: object | None = None, drop_params: bool = True, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ @@ -353,6 +401,7 @@ class AnthropicAdapter: custom_llm_provider=custom_llm_provider, api_base=api_base, deployment_model_info=deployment_model_info, + reasoning_summary=reasoning_summary, drop_params=drop_params, report_reasoning_drops=True, ) @@ -551,11 +600,12 @@ class LiteLLMAnthropicMessagesAdapter: custom_llm_provider: str | None = None, deployment_model_info: Mapping[str, object] | None = None, preserve_midturn_system: bool = False, + route: ChatRouteInputs = _NO_ROUTE_INPUTS, ) -> list[AllMessageValues]: new_messages: Final[list[AllMessageValues]] = [] replayable_messages: Final = ( messages - if _is_azure_responses_target(model, custom_llm_provider, deployment_model_info) + if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider, deployment_model_info, route) else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) ) leading_count: Final = next( @@ -1338,6 +1388,7 @@ class LiteLLMAnthropicMessagesAdapter: custom_llm_provider: str | None = None, api_base: str | None = None, deployment_model_info: Mapping[str, object] | None = None, + reasoning_summary: object | None = None, preserve_midturn_system: bool = False, drop_params: bool = True, report_reasoning_drops: bool = False, @@ -1351,27 +1402,15 @@ class LiteLLMAnthropicMessagesAdapter: for tools that exceeded OpenAI's 64-char limit """ # Debug: Processing Anthropic message request - new_messages: list[AllMessageValues] = [] tool_name_mapping: dict[str, str] = {} - ## CONVERT ANTHROPIC MESSAGES TO OPENAI messages_list: Final[list[AllAnthropicPassThroughMessageValues]] = cast( list[AllAnthropicPassThroughMessageValues], anthropic_message_request["messages"], ) - new_messages = self.translate_anthropic_messages_to_openai( - messages=messages_list, - model=anthropic_message_request.get("model"), - custom_llm_provider=custom_llm_provider, - deployment_model_info=deployment_model_info, - preserve_midturn_system=preserve_midturn_system, - ) - ## ADD SYSTEM MESSAGE TO MESSAGES - self._add_system_message_to_messages(new_messages, anthropic_message_request) - new_kwargs: Final[ChatCompletionRequest] = { "model": anthropic_message_request["model"], - "messages": new_messages, + "messages": [], } ## CONVERT METADATA (user_id + litellm metadata) self._translate_metadata_to_openai( @@ -1398,6 +1437,24 @@ class LiteLLMAnthropicMessagesAdapter: drop_params=drop_params, report_reasoning_drops=report_reasoning_drops, ) + ## CONVERT ANTHROPIC MESSAGES TO OPENAI, after tools and thinking pick the route + new_messages: Final = self.translate_anthropic_messages_to_openai( + messages=messages_list, + model=anthropic_message_request.get("model"), + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, + preserve_midturn_system=preserve_midturn_system, + route=ChatRouteInputs( + tools=new_kwargs.get("tools"), + reasoning_effort=new_kwargs.get("reasoning_effort"), + api_base=api_base, + thinking=anthropic_message_request.get("thinking"), + reasoning_summary=reasoning_summary, + ), + ) + ## ADD SYSTEM MESSAGE TO MESSAGES + self._add_system_message_to_messages(new_messages, anthropic_message_request) + new_kwargs["messages"] = new_messages ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( anthropic_message_request=anthropic_message_request, @@ -1502,9 +1559,9 @@ class LiteLLMAnthropicMessagesAdapter: for choice in choices ): new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks) - # Handle thinking blocks first - if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: - for thinking_block in choice.message.thinking_blocks: + signed_thinking_blocks = _anthropic_thinking_blocks(choice.message) + if signed_thinking_blocks: + for thinking_block in signed_thinking_blocks: if thinking_block.get("type") == "thinking": if is_empty_unsigned_thinking_block(thinking_block): continue @@ -1816,24 +1873,24 @@ class LiteLLMAnthropicMessagesAdapter: choice.delta ) is not None: return "text", TextBlock(type="text", text="") - elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): - thinking_blocks = choice.delta.thinking_blocks or [] - if len(thinking_blocks) > 0: - thinking_block = thinking_blocks[0] - if thinking_block["type"] == "redacted_thinking": - return "redacted_thinking", ChatCompletionRedactedThinkingBlock( - type="redacted_thinking", data=thinking_block.get("data") or "" - ) - if thinking_block["type"] == "thinking": - thinking = thinking_block.get("thinking") or "" - signature = thinking_block.get("signature") or "" + elif isinstance(choice, StreamingChoices) and (thinking_blocks := _streamed_thinking_blocks(choice.delta)): + thinking_block = thinking_blocks[0] + if thinking_block["type"] == "redacted_thinking": + redacted_data = thinking_block.get("data") + return "redacted_thinking", ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=redacted_data if isinstance(redacted_data, str) else "", + ) + if thinking_block["type"] == "thinking": + thinking = thinking_block.get("thinking") or "" + signature = thinking_block.get("signature") or "" - assert isinstance(thinking, str) - assert isinstance(signature, str) + assert isinstance(thinking, str) + assert isinstance(signature, str) - return "thinking", ChatCompletionThinkingBlock( - type="thinking", thinking=thinking if signature else "", signature=signature - ) + return "thinking", ChatCompletionThinkingBlock( + type="thinking", thinking=thinking if signature else "", signature=signature + ) # OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning # parsers) populate ``reasoning_content`` without ``thinking_blocks``. # ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the @@ -1866,19 +1923,17 @@ class LiteLLMAnthropicMessagesAdapter: for tool in choice.delta.tool_calls: if tool.function is not None and tool.function.arguments is not None: partial_json = (partial_json or "") + tool.function.arguments - elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): - thinking_blocks = choice.delta.thinking_blocks or [] - if len(thinking_blocks) > 0: - for thinking_block in thinking_blocks: - if thinking_block["type"] == "thinking": - thinking = thinking_block.get("thinking") or "" - signature = thinking_block.get("signature") or "" + elif isinstance(choice, StreamingChoices) and (thinking_blocks := _streamed_thinking_blocks(choice.delta)): + for thinking_block in thinking_blocks: + if thinking_block["type"] == "thinking": + thinking = thinking_block.get("thinking") or "" + signature = thinking_block.get("signature") or "" - assert isinstance(thinking, str) - assert isinstance(signature, str) + assert isinstance(thinking, str) + assert isinstance(signature, str) - reasoning_content += thinking - reasoning_signature += signature + reasoning_content += thinking + reasoning_signature += signature # Handle reasoning_content when thinking_blocks is not present # This handles providers like OpenRouter that return reasoning_content elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "reasoning_content"): diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 85f042f621c..fb1b6d5884b 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -19,13 +19,14 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.litellm_core_utils.asyncify import ( run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_with_readable_mode, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -43,7 +44,12 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import CountContextManagement from litellm.utils import ProviderConfigManager -from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator +from ..adapters.handler import ( + context_management_explicitly_dropped, + messages_without_rejected_reasoning, + request_safeguards_evaluator, + resent_without_rejected_reasoning, +) from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management from ..context_management.constants import COMPACT_EDIT_TYPE from ..context_management.dispatcher import normalize_spec @@ -391,6 +397,76 @@ def _build_responses_kwargs( return responses_kwargs +_ResponsesKwargsFor: TypeAlias = Callable[[AnthropicRequestMessages], Mapping[str, object]] +_ResponsesResult: TypeAlias = ResponsesAPIResponse | BaseResponsesAPIStreamingIterator +_SyncResponsesResult: TypeAlias = _ResponsesResult | Coroutine[object, object, _ResponsesResult] + + +def _is_encrypted_reasoning_item(item: object) -> bool: + mapping: Final = as_str_mapping(item) + return mapping is not None and mapping.get("type") == "reasoning" and bool(mapping.get("encrypted_content")) + + +def _sends_encrypted_reasoning(responses_kwargs: Mapping[str, object]) -> bool: + return any(map(_is_encrypted_reasoning_item, as_object_list(responses_kwargs.get("input")) or ())) + + +def _kwarg_text(responses_kwargs: Mapping[str, object], key: str) -> str: + value: Final = responses_kwargs.get(key) + return value if isinstance(value, str) else "" + + +def _messages_without_rejected_reasoning( + messages: AnthropicRequestMessages, + error: litellm.BadRequestError, + responses_kwargs: Mapping[str, object], + sent_encrypted_reasoning: bool, +) -> AnthropicRequestMessages | None: + return messages_without_rejected_reasoning( + messages, + error, + sent_encrypted_reasoning=sent_encrypted_reasoning, + provider=_kwarg_text(responses_kwargs, "custom_llm_provider"), + model=_kwarg_text(responses_kwargs, "model"), + ) + + +async def _aresponses_without_rejected_reasoning( + messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor +) -> tuple[Mapping[str, object], _ResponsesResult]: + responses_kwargs: Final = responses_kwargs_for(messages) + sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(responses_kwargs) + try: + with resent_without_rejected_reasoning(sent_encrypted_reasoning): + return responses_kwargs, await litellm.aresponses(**responses_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning( + messages, error, responses_kwargs, sent_encrypted_reasoning + ) + if replayed_messages is None: + raise + replay_kwargs: Final = responses_kwargs_for(replayed_messages) + return replay_kwargs, await litellm.aresponses(**replay_kwargs) + + +def _responses_without_rejected_reasoning( + messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor +) -> tuple[Mapping[str, object], _SyncResponsesResult]: + responses_kwargs: Final = responses_kwargs_for(messages) + sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(responses_kwargs) + try: + with resent_without_rejected_reasoning(sent_encrypted_reasoning): + return responses_kwargs, litellm.responses(**responses_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning( + messages, error, responses_kwargs, sent_encrypted_reasoning + ) + if replayed_messages is None: + raise + replay_kwargs: Final = responses_kwargs_for(replayed_messages) + return replay_kwargs, litellm.responses(**replay_kwargs) + + class LiteLLMMessagesToResponsesAPIHandler: """ Handles Anthropic /v1/messages requests for OpenAI / Azure models by @@ -422,28 +498,32 @@ class LiteLLMMessagesToResponsesAPIHandler: polyfill_result: Final = await _polyfilled( model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs ) - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=max_tokens, - messages=polyfill_result.messages if polyfill_result is not None else messages, - model=model, - context_management=native_context_management, - metadata=metadata, - output_config=output_config, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - output_format=output_format, - extra_kwargs=kwargs, - ) + + def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]: + return _build_responses_kwargs( + max_tokens=max_tokens, + messages=sent_messages, + model=model, + context_management=native_context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages) - result: Final = await litellm.aresponses(**responses_kwargs) + responses_kwargs, result = await _aresponses_without_rejected_reasoning( + polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( @@ -526,27 +606,31 @@ class LiteLLMMessagesToResponsesAPIHandler: else None ) polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=max_tokens, - messages=polyfill_result.messages if polyfill_result is not None else messages, - model=model, - context_management=native_context_management, - metadata=metadata, - output_config=output_config, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - output_format=output_format, - extra_kwargs=kwargs, - ) - result: Final = litellm.responses(**responses_kwargs) + def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]: + return _build_responses_kwargs( + max_tokens=max_tokens, + messages=sent_messages, + model=model, + context_management=native_context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + responses_kwargs, result = _responses_without_rejected_reasoning( + polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( diff --git a/litellm/main.py b/litellm/main.py index 1d18100d1d5..f3620d7a8c2 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -28,6 +28,7 @@ from functools import partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args +from litellm._internal_context import is_resent_failure from litellm._logging import redact_string from litellm._uuid import uuid @@ -6634,6 +6635,10 @@ def completion( ) +def _is_not_resent_failure(error: BaseException) -> bool: + return isinstance(error, Exception) and not is_resent_failure(error) + + def completion_with_retries(*args, **kwargs): """ Executes a litellm.completion() with 3 retries @@ -6653,10 +6658,15 @@ def completion_with_retries(*args, **kwargs): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.Retrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return retryer(original_function, *args, **kwargs) @@ -6679,10 +6689,15 @@ async def acompletion_with_retries(*args, **kwargs): retryer = tenacity.AsyncRetrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.AsyncRetrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return await retryer(original_function, *args, **kwargs) @@ -6707,10 +6722,15 @@ def responses_with_retries(*args, **kwargs): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.Retrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return retryer(original_function, *args, **kwargs) @@ -6734,10 +6754,15 @@ async def aresponses_with_retries(*args, **kwargs): retryer = tenacity.AsyncRetrying( wait=tenacity.wait_exponential(multiplier=1, max=10), stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), reraise=True, ) else: - retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True) + retryer = tenacity.AsyncRetrying( + stop=tenacity.stop_after_attempt(num_retries), + retry=tenacity.retry_if_exception(_is_not_resent_failure), + reraise=True, + ) return await retryer(original_function, *args, **kwargs) diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 826de8cfdc9..683a51fc6b6 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -4,7 +4,10 @@ from urllib.parse import urlsplit import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider, + resolve_model_and_provider_for_metadata, +) from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, @@ -291,3 +294,39 @@ def responses_api_bridge_check( model = model.replace("responses/", "") return model_info, model + + +def chat_completion_ends_in_responses_api( + model: str | None, + custom_llm_provider: str | None, + deployment_model_info: Mapping[str, object] | None = None, + *, + tools: Sequence[Mapping[str, object]] | None = None, + reasoning_effort: str | Mapping[str, object] | None = None, + reasoning_summary: object | None = None, + api_base: str | None = None, +) -> bool: + """Whether chat completion for this model is served by a Responses API. + + Encrypted reasoning can be replayed only there. A normal chat provider cannot + verify the block, so the messages adapter strips it. ``chatgpt``, a ``responses/`` + model, a deployment pinned to ``mode: responses``, and a request whose tools or + reasoning effort the bridge sends to Responses keep it. + """ + if not model: + return False + try: + resolved_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider) + except Exception as error: # noqa: BLE001 # unknown model names must not fail message translation + verbose_logger.debug("responses replay check failed for %s: %s", model, error) + return False + model_info, _resolved = responses_api_bridge_check( + model=resolved_model, + custom_llm_provider=provider, + tools=tools, + reasoning_effort=reasoning_effort, + reasoning_summary=reasoning_summary, + api_base=api_base, + deployment_model_info=deployment_model_info, + ) + return model_info.get("mode") == "responses" diff --git a/litellm/utils.py b/litellm/utils.py index 28e2440fbdd..b9bf872e853 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -53,7 +53,7 @@ import litellm.litellm_core_utils # audio_utils.utils is lazy-loaded - only imported when needed for transcription calls import litellm.litellm_core_utils.json_validation_rule -from litellm._internal_context import is_internal_call +from litellm._internal_context import is_internal_call, is_resent_failure from litellm._lazy_imports import ( _get_modified_max_tokens, get_default_encoding, @@ -2303,12 +2303,16 @@ def client(original_function): retry_policy=kwargs.get("retry_policy"), ) kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops - litellm.num_retries = None # set retries to None to prevent infinite loops + if not is_resent_failure(e): + litellm.num_retries = None # set retries to None to prevent infinite loops context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {}) is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e) + num_retries + and not is_completion_litellm_router_call + and not is_agentic_followup_failure(e) + and not is_resent_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2338,11 +2342,15 @@ def client(original_function): retry_policy=kwargs.get("retry_policy"), ) kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops - litellm.num_retries = None # set retries to None to prevent infinite loops + if not is_resent_failure(e): + litellm.num_retries = None # set retries to None to prevent infinite loops is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e) + num_retries + and not is_responses_litellm_router_call + and not is_agentic_followup_failure(e) + and not is_resent_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2355,7 +2363,7 @@ def client(original_function): end_time = datetime.datetime.now() # LOG FAILURE - handle streaming failure logging in the _next_ object, remove `handle_failure` once it's deprecated - if logging_obj: + if logging_obj and not is_resent_failure(e): logging_obj.failure_handler( e, traceback_exception, start_time, end_time ) # DO NOT MAKE THREADED - router retry fallback relies on this! @@ -2620,7 +2628,7 @@ def client(original_function): # Reuse the timestamp taken right when the deployment call itself failed, before # the failure hook ran, so a slow callback doesn't inflate the reported duration. end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with - if logging_obj and not _is_litellm_internal_call: + if logging_obj and not _is_litellm_internal_call and not is_resent_failure(e): unbind_budget_reservation_from_callbacks(logging_obj.litellm_params) try: logging_obj.failure_handler( @@ -2645,6 +2653,7 @@ def client(original_function): and not is_acompletion_litellm_router_call and not isinstance(e, ImportError) and not is_agentic_followup_failure(e) + and not is_resent_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops @@ -2655,8 +2664,9 @@ def client(original_function): elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" result = await litellm.acompletion_with_retries(*args, **kwargs) - except Exception: - pass + except Exception as retry_error: + if is_resent_failure(retry_error): + raise else: return result elif ( @@ -2679,6 +2689,7 @@ def client(original_function): and not is_aresponses_litellm_router_call and not isinstance(e, ImportError) and not is_agentic_followup_failure(e) + and not is_resent_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops @@ -2689,8 +2700,9 @@ def client(original_function): elif isinstance(e, openai.APIError): # generic api error kwargs["retry_strategy"] = "constant_retry" result = await litellm.aresponses_with_retries(*args, **kwargs) - except Exception: - pass + except Exception as retry_error: + if is_resent_failure(retry_error): + raise else: return result diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 19aefb430c7..7ba833aad14 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3837,6 +3837,386 @@ def test_reasoning_items_streaming_emitted_on_response_completed(): assert ri["summary"][0]["text"] == summary_text +def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keeps(): + """A chatgpt /v1/messages stream drops the reasoning item unless this delta carries thinking_blocks. + + response.output_item.done is the only event that has encrypted_content. The chat delta has to + expose it as a signed thinking block, and that block alone has to count as stream content. + """ + from unittest.mock import MagicMock + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter + + encrypted: Final = "gAAAA_round_trip" + done: Final = { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "reasoning", + "id": "rs_done", + "encrypted_content": encrypted, + "summary": [], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True + ).chunk_parser(done) + blocks: Final = result.choices[0].delta.thinking_blocks + assert blocks is not None + assert blocks[0]["type"] == "redacted_thinking" + assert blocks[0]["data"] == encrypted_reasoning_signature(encrypted) + assert blocks[0]["index"] == 0 + assert getattr(result.choices[0].delta, "reasoning_content", None) in (None, "") + + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model="chatgpt/gpt-6.1-sol", + logging_obj=MagicMock(), + custom_llm_provider="chatgpt", + ) + assert wrapper.is_chunk_non_empty({}, result, {}) is True + + block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block( + choices=result.choices + ) + assert block_type == "redacted_thinking" + assert start["data"] == encrypted_reasoning_signature(encrypted) + + +_SIGNED_REASONING_STREAM_BODY: Final[Mapping[str, object]] = { + **_searched_responses_body("azure/gpt-6-sol"), + "output": [ + {"type": "reasoning", "id": "rs_signed", "summary": [], "encrypted_content": "gAAAA_consumer"}, + _PONG_MESSAGE, + ], +} + + +def _signed_reasoning_sse() -> str: + events: Final = ( + { + "type": "response.created", + "response": {**_SIGNED_REASONING_STREAM_BODY, "status": "in_progress", "output": [], "usage": None}, + }, + {"type": "response.output_item.done", "output_index": 0, "item": _SIGNED_REASONING_STREAM_BODY["output"][0]}, + { + "type": "response.output_text.delta", + "item_id": "msg_logged_call_type", + "output_index": 1, + "content_index": 0, + "delta": "pong", + }, + {"type": "response.completed", "response": _SIGNED_REASONING_STREAM_BODY}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +_AZURE_BRIDGE_KWARGS: Final[Mapping[str, object]] = { + "api_base": "https://example.invalid", + "api_version": "2025-04-01-preview", + "api_key": "test-key", +} + + +@pytest.mark.asyncio +async def test_streamed_reasoning_signature_reaches_messages_clients_and_never_chat_completions_deltas( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://example.invalid/openai/responses").mock( + side_effect=lambda _request: httpx.Response( + 200, text=_signed_reasoning_sse(), headers={"content-type": "text/event-stream"} + ) + ) + chat_stream: Final = await litellm.acompletion( + model="azure/gpt-6-sol", + messages=[{"role": "user", "content": "ping"}], + tools=[_GET_WEATHER_TOOL], + stream=True, + **_AZURE_BRIDGE_KWARGS, + ) + chat_deltas: Final = [chunk.choices[0].delta async for chunk in chat_stream if chunk.choices] + messages_stream: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-6-sol", + messages=[{"role": "user", "content": "ping"}], + tools=[{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}], + max_tokens=64, + stream=True, + **_AZURE_BRIDGE_KWARGS, + ) + sse: Final = b"".join([chunk async for chunk in messages_stream]).decode() + + assert [getattr(delta, "thinking_blocks", None) for delta in chat_deltas if getattr(delta, "thinking_blocks", None)] == [] + assert "".join(delta.content or "" for delta in chat_deltas) == "pong" + messages_events: Final = [json.loads(line[len("data: ") :]) for line in sse.splitlines() if line.startswith("data: ")] + started: Final = [event["content_block"] for event in messages_events if event.get("type") == "content_block_start"] + assert [block["type"] for block in started] == ["redacted_thinking", "text"] + assert started[0]["data"] == encrypted_reasoning_signature("gAAAA_consumer") + + +def test_reasoning_output_item_done_with_summary_signs_the_thinking_text(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + + encrypted: Final = "gAAAA_with_summary" + done: Final = { + "type": "response.output_item.done", + "output_index": 2, + "item": { + "type": "reasoning", + "id": "rs_sum", + "encrypted_content": encrypted, + "summary": [{"type": "summary_text", "text": "Oslo is colder."}], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True + ).chunk_parser(done) + block: Final = result.choices[0].delta.thinking_blocks[0] + assert block["type"] == "thinking" + assert block["thinking"] == "Oslo is colder." + assert block["signature"] == encrypted_reasoning_signature(encrypted) + assert block["index"] == 2 + + +def test_streamed_reasoning_summary_reaches_messages_clients_once_with_its_signature(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicStreamWrapper + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + encrypted: Final = "gAAAA_streamed_summary" + events: Final = ( + {"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 0}, + {"type": "response.reasoning_summary_text.delta", "output_index": 0, "delta": "Oslo is "}, + {"type": "response.reasoning_summary_text.delta", "output_index": 0, "delta": "colder."}, + { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "reasoning", + "id": "rs_streamed", + "encrypted_content": encrypted, + "summary": [{"type": "summary_text", "text": "Oslo is colder."}], + }, + }, + {"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."}, + ) + parser: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True + ) + chunks: Final = ( + *(parser.chunk_parser(event) for event in events), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]), + ) + + sse_events: Final = list(AnthropicStreamWrapper(completion_stream=iter(chunks), model="chatgpt/gpt-5.5")) + deltas: Final = [event["delta"] for event in sse_events if event.get("type") == "content_block_delta"] + assert "".join(d.get("thinking", "") for d in deltas if d["type"] == "thinking_delta") == "Oslo is colder." + assert [d["signature"] for d in deltas if d["type"] == "signature_delta"] == [ + encrypted_reasoning_signature(encrypted) + ] + + +def test_reasoning_item_repeated_at_stream_end_reaches_messages_clients_once(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicStreamWrapper + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + encrypted: Final = "gAAAA_completed_repeat" + reasoning_item: Final = {"type": "reasoning", "id": "rs_repeat", "summary": [], "encrypted_content": encrypted} + events: Final = ( + {"type": "response.output_item.done", "output_index": 0, "item": reasoning_item}, + {"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."}, + ) + parser: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True + ) + chunks: Final = ( + *(parser.chunk_parser(event) for event in events), + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(reasoning_items=[reasoning_item]), finish_reason=None)] + ), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]), + ) + + sse_events: Final = list(AnthropicStreamWrapper(completion_stream=iter(chunks), model="azure/gpt-5.4-nano")) + started: Final = [event["content_block"] for event in sse_events if event.get("type") == "content_block_start"] + assert [block["type"] for block in started] == ["redacted_thinking", "text"] + assert started[0]["data"] == encrypted_reasoning_signature(encrypted) + + +def test_reasoning_output_item_done_without_encrypted_content_stays_empty(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + done: Final = { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "reasoning", + "id": "rs_plain", + "summary": [{"type": "summary_text", "text": "already streamed"}], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True, streams_reasoning_signatures=True + ).chunk_parser(done) + assert getattr(result.choices[0].delta, "thinking_blocks", None) in (None, []) + + +def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only(): + from unittest.mock import Mock + + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.llms.anthropic.messages.adapters.transformation import ( + LiteLLMAnthropicMessagesAdapter, + ) + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import ModelResponse, Usage + + encrypted: Final = "gAAAA_non_stream" + reasoning: Final = ResponseReasoningItem( + id="rs_tool", + summary=[], + type="reasoning", + content=None, + encrypted_content=encrypted, + status=None, + ) + function_call: Final = ResponseFunctionToolCall( + id="fc_1", + type="function_call", + status="completed", + arguments='{"city":"Oslo"}', + call_id="call_1", + name="get_weather", + ) + raw: Final = ResponsesAPIResponse.model_construct( + id="resp_tool", + created_at=1, + error=None, + incomplete_details=None, + model="gpt-6.1-sol", + output=[reasoning, function_call], + status="completed", + usage=None, + ) + model_response: Final = ModelResponse( + id="chatcmpl-tool", + created=1, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ) + result: Final = LiteLLMResponsesTransformationHandler().transform_response( + model="gpt-6.1-sol", + raw_response=raw, + model_response=model_response, + logging_obj=Mock(), + request_data={"model": "gpt-6.1-sol"}, + messages=[{"role": "user", "content": "Which city?"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + message: Final = result.choices[0].message + assert message.reasoning_items[0]["encrypted_content"] == encrypted + assert "thinking_blocks" not in result.model_dump()["choices"][0]["message"] + assert message.tool_calls[0].function.name == "get_weather" + + anthropic_response: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( + response=result, tool_name_mapping=None + ) + content: Final = anthropic_response["content"] + assert [block["type"] for block in content] == ["redacted_thinking", "tool_use"] + assert content[0]["data"] == encrypted_reasoning_signature(encrypted) + + +@pytest.mark.parametrize( + ("call_type", "kept_ids"), + [ + pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client_keeps_every_item"), + pytest.param("acompletion", ["rs_second"], id="chat_client_keeps_the_last_item"), + ], +) +def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item( + call_type: str, kept_ids: list[str] +) -> None: + from types import SimpleNamespace + from unittest.mock import Mock + + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import ModelResponse, Usage + + def reasoning(item_id: str) -> ResponseReasoningItem: + return ResponseReasoningItem( + id=item_id, summary=[], type="reasoning", content=None, encrypted_content=f"gAAAA_{item_id}", status=None + ) + + def call(index: int, city: str) -> ResponseFunctionToolCall: + return ResponseFunctionToolCall( + id=f"fc_{index}", + type="function_call", + status="completed", + arguments=json.dumps({"city": city}), + call_id=f"call_{index}", + name="get_weather", + ) + + raw: Final = ResponsesAPIResponse.model_construct( + id="resp_two_reasoning", + created_at=1, + error=None, + incomplete_details=None, + model="gpt-5.4-nano", + output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")], + status="completed", + usage=None, + ) + result: Final = LiteLLMResponsesTransformationHandler().transform_response( + model="gpt-5.4-nano", + raw_response=raw, + model_response=ModelResponse( + id="chatcmpl-two-reasoning", + created=1, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ), + logging_obj=SimpleNamespace(call_type=call_type), + request_data={"model": "gpt-5.4-nano"}, + messages=[{"role": "user", "content": "Oslo or Lyon?"}], + optional_params={}, + litellm_params={"custom_llm_provider": "azure"}, + encoding=Mock(), + ) + + message: Final = result.choices[0].message + assert [item["id"] for item in message.reasoning_items] == kept_ids + assert [tool_call.function.name for tool_call in message.tool_calls] == ["get_weather", "get_weather"] + + def test_streaming_function_call_tool_id_for_degenerate_call_id(): """In streaming, Bedrock Mantle's function_call event carries a unique ``id`` (``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 94fd915844a..df50ae9dc44 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -276,6 +276,37 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp ) +def test_is_chunk_non_empty_keeps_thinking_blocks_without_other_content( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """Encrypted reasoning arrives as thinking_blocks and nothing else, so that delta has to be yielded.""" + chunk: Final = ModelResponseStream( + id="chunk-thinking", + object="chat.completion.chunk", + created=1, + model="chatgpt/gpt-6.1-sol", + choices=[ + StreamingChoices( + index=0, + finish_reason=None, + delta=Delta( + thinking_blocks=[ + {"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"} + ] + ), + ) + ], + ) + assert ( + initialized_custom_stream_wrapper.is_chunk_non_empty( + completion_obj={}, + model_response=chunk, + response_obj={}, + ) + is True + ) + + def test_is_chunk_non_empty_with_annotations( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index 420de9cb4f5..bad7b3f1750 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -2,7 +2,8 @@ import base64 import functools import json import logging -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from contextlib import nullcontext from types import MappingProxyType from typing import Any, Final, cast @@ -11,6 +12,7 @@ import pytest import respx import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, ) @@ -34,6 +36,7 @@ from litellm.llms.anthropic.messages.native.mid_conversation_system import ( from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.anthropic import ( AnthopicMessagesAssistantMessageParam, + AnthropicMessagesRequest, AnthropicMessagesUserMessageParam, ) from litellm.types.llms.openai import ChatCompletionAssistantToolCall @@ -616,6 +619,218 @@ def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"] +def test_chatgpt_messages_keep_encrypted_reasoning_and_replay_it(): + """chatgpt /v1/messages ends in the Responses API, so the signed block has to survive into the next request.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + summary_encrypted: Final = "gAAAA_summary" + bytes_encrypted: Final = "gAAAA_bytes" + messages: Final = [ + AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + { + "type": "thinking", + "thinking": "Oslo is colder.", + "signature": encrypted_reasoning_signature(summary_encrypted), + }, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(bytes_encrypted)}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}, + ], + ), + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}], + ), + ] + + kept: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="chatgpt/gpt-6.1-sol", + custom_llm_provider="chatgpt", + ) + assistant: Final = kept[1] + assert [block["type"] for block in assistant["thinking_blocks"]] == ["thinking", "redacted_thinking"] + assert assistant["thinking_blocks"][0]["signature"] == encrypted_reasoning_signature(summary_encrypted) + assert assistant["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted) + + responses_input, _instructions = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api( + kept + ) + reasoning: Final = tuple(item for item in responses_input if item.get("type") == "reasoning") + assert [item["encrypted_content"] for item in reasoning] == [summary_encrypted, bytes_encrypted] + types: Final = [item.get("type") for item in responses_input] + assert types.index("reasoning") < types.index("function_call") + + stripped: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="gpt-4o", + custom_llm_provider="openai", + ) + assert not stripped[1].get("thinking_blocks") + + prefixed: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="responses/gpt-5-mini", + custom_llm_provider="openai", + ) + assert prefixed[1]["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted) + + +def test_translate_openai_reasoning_items_become_signed_thinking_blocks(): + encrypted: Final = "gAAAA_outbound" + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content="Oslo.", + reasoning_items=[ + { + "type": "reasoning", + "id": "rs_1", + "encrypted_content": encrypted, + "summary": [{"type": "summary_text", "text": "Oslo is colder."}], + } + ], + tool_calls=[ + ChatCompletionAssistantToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments='{"city": "Oslo"}'), + ) + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert result[0]["type"] == "thinking" + assert result[0]["thinking"] == "Oslo is colder." + assert result[0]["signature"] == encrypted_reasoning_signature(encrypted) + assert result[1]["type"] == "text" + assert result[2]["type"] == "tool_use" + + +def test_translate_openai_reasoning_items_without_summary_become_redacted_thinking(): + encrypted: Final = "gAAAA_bytes_only" + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content=None, + reasoning_items=[ + {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert result == [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(encrypted)}, + ] + + +def test_translate_openai_reasoning_items_are_not_duplicated_when_thinking_blocks_already_carry_them(): + encrypted: Final = "gAAAA_once" + signature: Final = encrypted_reasoning_signature(encrypted) + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content="Oslo.", + thinking_blocks=[{"type": "redacted_thinking", "data": signature}], + reasoning_items=[ + {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert [block["type"] for block in result] == ["redacted_thinking", "text"] + + +def _replayed_reasoning_request(tools: list[dict[str, object]]) -> AnthropicMessagesRequest: + return AnthropicMessagesRequest( + model="azure/gpt-5.4-nano", + max_tokens=64, + messages=[ + AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_azure_tools")}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}, + ], + ), + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}], + ), + ], + tools=tools, + ) + + +@pytest.mark.parametrize( + ("tools", "kept"), + [ + pytest.param( + [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}], + True, + id="function_tools_bridge_to_responses", + ), + pytest.param([], False, id="no_tools_stay_on_chat"), + ], +) +def test_encrypted_reasoning_is_kept_when_the_request_tools_route_it_to_responses( + tools: list[dict[str, object]], kept: bool +) -> None: + openai_request, _mapping = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + _replayed_reasoning_request(tools), + custom_llm_provider="azure", + api_base="https://keep-check.openai.azure.com", + ) + + assistant: Final = openai_request["messages"][1] + assert bool(assistant.get("thinking_blocks")) is kept + + +@pytest.mark.parametrize( + ("deployment_params", "kept"), + [ + pytest.param({"reasoning_summary": "auto"}, True, id="deployment_summary_bridges_to_responses"), + pytest.param({"extra_body": {"reasoningSummary": "auto"}}, True, id="extra_body_summary_bridges_to_responses"), + pytest.param({}, False, id="no_summary_stays_on_chat"), + ], +) +def test_encrypted_reasoning_is_kept_when_the_deployment_reasoning_summary_routes_it_to_responses( + deployment_params: dict[str, object], kept: bool +) -> None: + from litellm.llms.anthropic.messages.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + request: Final = _replayed_reasoning_request([]) + completion_kwargs, _mapping = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=64, + messages=cast(list[dict[str, object]], request["messages"]), + model="azure/gpt-5.1", + thinking={"type": "enabled", "budget_tokens": 2048}, + extra_kwargs={"api_base": "https://keep-check.openai.azure.com", **deployment_params}, + ) + + assistant: Final = cast(list[Mapping[str, object]], completion_kwargs["messages"])[1] + assert bool(assistant.get("thinking_blocks")) is kept + + def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): """Reasoning-aware chat providers read reasoning_content, so thinking text must land there. @@ -5578,3 +5793,503 @@ async def test_chat_bridge_tool_result_text_block_without_text_reaches_the_model ], } ] + + +_RETRY_API_BASE: Final = "https://encrypted-retry.openai.azure.com" +_RETRY_URL: Final = r".*encrypted-retry\.openai\.azure\.com/openai/.*responses.*" +_RETRY_TOOLS: Final = [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}] +_RETRY_REJECTION: Final = httpx.Response( + 400, + json={ + "error": { + "message": "The encrypted content gAAA_foreign could not be verified.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_encrypted_content", + } + }, +) +_UNRELATED_REJECTION: Final = httpx.Response( + 400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}} +) +_UNRELATED_UNVERIFIED_REJECTION: Final = httpx.Response( + 400, json={"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}} +) +_RETRY_REPLY: Final = { + "id": "resp_retry", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.4-nano", + "output": [ + { + "type": "message", + "id": "msg_retry", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Oslo", "annotations": []}], + } + ], + "usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12}, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], +} + + +def _history_with_foreign_reasoning() -> list[dict[str, object]]: + return [ + {"role": "user", "content": "Which city is colder?"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}]}, + ] + + +def _sent_reasoning_items(request: httpx.Request) -> list[object]: + sent: Final = json.loads(request.read()) + return [item for item in sent["input"] if item.get("type") == "reasoning"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, + caplog: pytest.LogCaptureFixture, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + warnings: Final = tuple( + record.getMessage() + for record in caplog.records + if record.levelno >= logging.WARNING and record.name == "LiteLLM" + ) + retry_warnings: Final = tuple(message for message in warnings if "retrying once without it" in message) + assert route.call_count == 2 + assert len(_sent_reasoning_items(route.calls[0].request)) == 1 + assert _sent_reasoning_items(route.calls[1].request) == [] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + assert retry_warnings == ( + "Backend rejected replayed encrypted reasoning, retrying once without it. " + "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " + "provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content", + ) + assert all( + "gAAA_foreign" not in message and "litellm_encrypted_reasoning:" not in message for message in warnings + ) + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + assert route.call_count == 2 + assert _sent_reasoning_items(route.calls[1].request) == [] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_async_messages_with_num_retries_send_the_rejected_reasoning_only_once( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 0] + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +def test_sync_messages_with_global_num_retries_send_the_rejected_reasoning_once_and_keep_the_setting( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "num_retries", 2) + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 0] + assert litellm.num_retries == 2 + + +_SERVER_ERROR: Final = httpx.Response( + 500, json={"error": {"message": "The server had an error while processing your request.", "type": "server_error"}} +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_async_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_once_rejected( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_SERVER_ERROR, _RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 1, 0] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +def test_sync_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_once_rejected( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock( + side_effect=[_SERVER_ERROR, _RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)] + ) + + response: Final = litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + num_retries=2, + ) + + assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 1, 0] + assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] + + +class _FailureLog(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.rejections: tuple[str, ...] = () + + def _record(self, kwargs: Mapping[str, object]) -> None: + error: Final = str(kwargs.get("exception")) + self.rejections = ( + *self.rejections, + next((phrase for phrase in _UPSTREAM_REJECTION_PHRASES if phrase in error), error), + ) + + def log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs) + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs) + + +_UPSTREAM_REJECTION_PHRASES: Final = ("could not be verified", "max_output_tokens is too small") +_OUTCOMES_OF_THE_TURN_SENT_AGAIN: Final = ( + pytest.param(httpx.Response(200, json=_RETRY_REPLY), (), id="served"), + pytest.param(_UNRELATED_REJECTION, ("max_output_tokens is too small",), id="rejected"), +) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN) +async def test_async_messages_log_only_the_outcome_of_the_turn_sent_again( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sent_again: httpx.Response, + logged_rejections: tuple[str, ...], +) -> None: + failure_log: Final = _FailureLog() + monkeypatch.setattr(litellm, "callbacks", [failure_log]) + respx_mock.post(url__regex=_RETRY_URL).mock(side_effect=[_RETRY_REJECTION, sent_again]) + + with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): + await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + assert failure_log.rejections == logged_rejections + + +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN) +def test_sync_messages_log_only_the_outcome_of_the_turn_sent_again( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sent_again: httpx.Response, + logged_rejections: tuple[str, ...], +) -> None: + failure_log: Final = _FailureLog() + monkeypatch.setattr(litellm, "callbacks", [failure_log]) + respx_mock.post(url__regex=_RETRY_URL).mock(side_effect=[_RETRY_REJECTION, sent_again]) + + with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): + litellm.anthropic.messages.create( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=_history_with_foreign_reasoning(), + ) + + assert failure_log.rejections == logged_rejections + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +@pytest.mark.parametrize( + ("messages", "rejection"), + [ + pytest.param( + [{"role": "user", "content": "Which city is colder?"}], + _RETRY_REJECTION, + id="no_encrypted_reasoning_to_drop", + ), + pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"), + pytest.param( + _history_with_foreign_reasoning(), _UNRELATED_UNVERIFIED_REJECTION, id="unrelated_unverified_rejection" + ), + ], +) +async def test_messages_do_not_retry_other_rejections( + respx_mock: respx.MockRouter, messages: list[dict[str, object]], rejection: httpx.Response +) -> None: + route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(return_value=rejection) + + with pytest.raises(litellm.BadRequestError): + await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + messages=messages, + ) + + assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_messages_do_not_retry_when_the_translated_request_already_dropped_the_reasoning( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(url__regex=_CHAT_BRIDGE_URL).mock(return_value=_RETRY_REJECTION) + + with pytest.raises(litellm.BadRequestError): + await litellm.anthropic.messages.acreate( + model=_CHAT_BRIDGE_MODEL, + api_base=_CHAT_BRIDGE_API_BASE, + api_key="fake-key", + max_tokens=64, + messages=_history_with_foreign_reasoning(), + ) + + assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_openai_thinking_on_the_chat_adapter_replays_the_reasoning_its_responses_route_minted( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True) + route: Final = respx_mock.post(url__regex=r".*openai-thinking\.example/v1/responses").mock( + return_value=httpx.Response(200, json=_RETRY_REPLY) + ) + + await litellm.anthropic.messages.acreate( + model="openai/gpt-5.4-nano", + api_base="https://openai-thinking.example/v1", + api_key="fake-key", + max_tokens=2048, + thinking={"type": "enabled", "budget_tokens": 1024}, + messages=[ + {"role": "user", "content": "Which city is colder?"}, + { + "role": "assistant", + "content": [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")}, + {"type": "text", "text": "Oslo."}, + ], + }, + {"role": "user", "content": "And by how much?"}, + ], + ) + + assert route.call_count == 1 + assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] + + +def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes: + return b"".join( + f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode() + for number, event in enumerate(events) + ) + + +def _reasoning_item(item_id: str, encrypted: str, summary: str = "") -> dict[str, object]: + return { + "type": "reasoning", + "id": item_id, + "summary": [{"type": "summary_text", "text": summary}] if summary else [], + "encrypted_content": encrypted, + } + + +def _streamed_blocks(frames: str) -> list[tuple[str, str, str | None]]: + events: Final = tuple( + json.loads(line.removeprefix("data: ")) for line in frames.splitlines() if line.startswith("data: ") + ) + deltas: Final = tuple(event for event in events if event["type"] == "content_block_delta") + + def block_deltas(index: int) -> tuple[Mapping[str, str], ...]: + return tuple(delta["delta"] for delta in deltas if delta["index"] == index) + + return [ + ( + event["content_block"]["type"], + "".join(delta.get("thinking", "") for delta in block_deltas(event["index"])), + next( + (delta["signature"] for delta in block_deltas(event["index"]) if delta["type"] == "signature_delta"), + None, + ), + ) + for event in events + if event["type"] == "content_block_start" + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("_chat_bridge_httpx") +async def test_streamed_reasoning_summaries_stay_with_their_own_encrypted_item( + respx_mock: respx.MockRouter, +) -> None: + response: Final = {**_RETRY_REPLY, "id": "resp_two_summaries", "status": "in_progress", "output": []} + message: Final = { + "type": "message", + "id": "msg_two_summaries", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Oslo", "annotations": []}], + } + first: Final = _reasoning_item("rs_1", "gAAA_first", "Oslo is further north.") + second: Final = _reasoning_item("rs_2", "gAAA_second", "Check the forecast.") + completed: Final = {**response, "status": "completed", "output": [first, second, message]} + + def summary_delta(output_index: int, item_id: str, text: str) -> dict[str, object]: + return { + "type": "response.reasoning_summary_text.delta", + "output_index": output_index, + "item_id": item_id, + "summary_index": 0, + "delta": text, + } + + events: Final = ( + {"type": "response.created", "response": response}, + {"type": "response.output_item.added", "output_index": 0, "item": _reasoning_item("rs_1", "")}, + summary_delta(0, "rs_1", "Oslo is further north."), + {"type": "response.output_item.done", "output_index": 0, "item": first}, + {"type": "response.output_item.added", "output_index": 1, "item": _reasoning_item("rs_2", "")}, + summary_delta(1, "rs_2", "Check the forecast."), + {"type": "response.output_item.done", "output_index": 1, "item": second}, + {"type": "response.output_item.added", "output_index": 2, "item": {**message, "content": []}}, + { + "type": "response.output_text.delta", + "output_index": 2, + "item_id": "msg_two_summaries", + "content_index": 0, + "delta": "Oslo", + }, + {"type": "response.output_item.done", "output_index": 2, "item": message}, + {"type": "response.completed", "response": completed}, + ) + respx_mock.post(url__regex=_RETRY_URL).mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events)) + ) + + stream: Final = await litellm.anthropic.messages.acreate( + model="azure/gpt-5.4-nano", + api_base=_RETRY_API_BASE, + api_key="fake-key", + api_version="2025-04-01-preview", + max_tokens=64, + tools=_RETRY_TOOLS, + stream=True, + messages=[{"role": "user", "content": "Which city is colder?"}], + ) + frames: Final = b"".join([chunk async for chunk in stream]).decode() + + assert _streamed_blocks(frames) == [ + ("thinking", "Oslo is further north.", encrypted_reasoning_signature("gAAA_first")), + ("thinking", "Check the forecast.", encrypted_reasoning_signature("gAAA_second")), + ("text", "", None), + ] diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index c06c9d721ac..cd6e323daa2 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -3,20 +3,24 @@ import json import os import sys from collections.abc import AsyncIterator, Mapping +from contextlib import nullcontext from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch +import httpx import pytest import respx sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import encrypted_reasoning_signature from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, @@ -1135,3 +1139,291 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( ] message_delta: Final = next(event for event in events if event["type"] == "message_delta") assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +REJECTED_REPLAYED_REASONING: Final = { + "error": { + "message": "The encrypted content gAAA_foreign could not be verified.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_encrypted_content", + } +} +UNVERIFIED_ORGANIZATION: Final = { + "error": {"message": "The organization could not be verified.", "type": "invalid_request_error"} +} + + +def _history(*, replayed_reasoning: bool) -> list[dict[str, object]]: + reasoning: Final = ( + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},) + if replayed_reasoning + else () + ) + return [ + {"role": "user", "content": "List the logs."}, + {"role": "assistant", "content": [*reasoning, {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "app.log"}]}, + ] + + +def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]: + return [ + sum(item.get("type") == "reasoning" for item in json.loads(call.request.content)["input"]) + for call in upstream.calls + ] + + +def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool): + return _upstream_after(respx_mock, monkeypatch, (first,), stream) + + +def _upstream_after( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, failures: tuple[httpx.Response, ...], stream: bool +) -> respx.Route: + monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + served: Final = ( + httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + if stream + else httpx.Response(200, json=TEXT_RESPONSE) + ) + return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[*failures, served]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), stream) + + response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=stream, + custom_llm_provider="openai", + ) + events: Final = ( + [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], response)] + if stream + else [cast(Mapping[str, object], response)] + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert events[-1]["type"] in ("message", "message_stop") + + +def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) + + response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] + + +@pytest.mark.asyncio +async def test_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) + + await litellm.anthropic.messages.acreate( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + + +def test_sync_openai_turn_with_global_num_retries_sends_the_rejected_reasoning_once_and_keeps_the_setting( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "num_retries", 2) + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) + + litellm.anthropic.messages.create( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert litellm.num_retries == 2 + + +_SERVER_ERROR: Final = { + "error": {"message": "The server had an error while processing your request.", "type": "server_error"} +} + + +@pytest.mark.asyncio +async def test_openai_turn_retries_a_server_error_with_the_reasoning_and_drops_it_only_once_rejected( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream_after( + respx_mock, + monkeypatch, + (httpx.Response(500, json=_SERVER_ERROR), httpx.Response(400, json=REJECTED_REPLAYED_REASONING)), + False, + ) + + response: Final = await litellm.anthropic.messages.acreate( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] + + +def test_sync_openai_turn_retries_a_server_error_with_the_reasoning_and_drops_it_only_once_rejected( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream_after( + respx_mock, + monkeypatch, + (httpx.Response(500, json=_SERVER_ERROR), httpx.Response(400, json=REJECTED_REPLAYED_REASONING)), + False, + ) + + response: Final = litellm.anthropic.messages.create( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + num_retries=2, + ) + + assert _replayed_reasoning_counts(upstream) == [1, 1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] + + +class _FailureLog(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.rejections: tuple[str, ...] = () + + def _record(self, kwargs: Mapping[str, object]) -> None: + error: Final = str(kwargs.get("exception")) + self.rejections = ( + *self.rejections, + next((phrase for phrase in ("encrypted content", "organization") if phrase in error), error), + ) + + def log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs) + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs) + + +_OUTCOMES_OF_THE_TURN_SENT_AGAIN: Final = ( + pytest.param(httpx.Response(200, json=TEXT_RESPONSE), (), id="served"), + pytest.param(httpx.Response(400, json=UNVERIFIED_ORGANIZATION), ("organization",), id="rejected"), +) + + +def _rejected_then(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sent_again: httpx.Response) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.openai.com/v1/responses").mock( + side_effect=[httpx.Response(400, json=REJECTED_REPLAYED_REASONING), sent_again] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN) +async def test_openai_turn_logs_only_the_outcome_of_the_turn_sent_again( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sent_again: httpx.Response, + logged_rejections: tuple[str, ...], +) -> None: + failure_log: Final = _FailureLog() + monkeypatch.setattr(litellm, "callbacks", [failure_log]) + _rejected_then(respx_mock, monkeypatch, sent_again) + + with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): + await litellm.anthropic.messages.acreate( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + ) + + assert failure_log.rejections == logged_rejections + + +@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN) +def test_sync_openai_turn_logs_only_the_outcome_of_the_turn_sent_again( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sent_again: httpx.Response, + logged_rejections: tuple[str, ...], +) -> None: + failure_log: Final = _FailureLog() + monkeypatch.setattr(litellm, "callbacks", [failure_log]) + _rejected_then(respx_mock, monkeypatch, sent_again) + + with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext(): + litellm.anthropic.messages.create( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + ) + + assert failure_log.rejections == logged_rejections + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "replayed_reasoning, rejection", + [ + pytest.param(False, REJECTED_REPLAYED_REASONING, id="no_replayed_reasoning"), + pytest.param(True, UNVERIFIED_ORGANIZATION, id="unrelated_rejection"), + ], +) +async def test_openai_turn_is_not_sent_again_for_other_rejections( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + replayed_reasoning: bool, + rejection: Mapping[str, object], +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=rejection), False) + + with pytest.raises(litellm.BadRequestError): + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=replayed_reasoning), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + ) + + assert upstream.call_count == 1 diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 2a5c3c81f18..548f78d148b 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -8,7 +8,7 @@ import logging import os import threading import urllib.parse -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime @@ -9395,6 +9395,19 @@ async def test_responses_with_retries(sync_mode): assert mock_responses.call_args.kwargs["max_retries"] == 0 +@pytest.mark.parametrize("retry_function", [acompletion_with_retries, aresponses_with_retries]) +@pytest.mark.asyncio +async def test_async_retry_helpers_stop_when_the_call_is_cancelled( + retry_function: Callable[..., Awaitable[object]], +) -> None: + upstream: Final = AsyncMock(side_effect=[asyncio.CancelledError(), MagicMock()]) + + with pytest.raises(asyncio.CancelledError): + await retry_function(model="gpt-4o", num_retries=3, original_function=upstream) + + assert upstream.await_count == 1 + + def test_azure_embedding_exceptions(): with pytest.raises(Exception, match="Mock error") as exc_info: litellm.embedding( From 9ec3b5fd8bc07706e674e5f936212e6ddd723a9f Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 7 Nov 2026 11:56:47 -0800 Subject: [PATCH 09/17] feat(cost-map): move -latest model aliases into the cost map (#50491) * feat(cost-map): move -latest model aliases into the cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: load the finalized local cost map in the moonshot alias test and assert alias pricing by behavior Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: compare alias cost map rows to their targets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(cost-map): give each -latest alias its own alias_of row Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): keep alias_of rows out of alias lists and add them to the registry schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 91e6cf2c27011d45b9f96e941c3189098df9c4f1) * fix(cost-map): let alias lists win over alias_of rows in both loaders and never serve a malformed pointer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit a54c42218a0e306b2ad4cd1aea9ad70f54392e8f) * feat(cost-map): add -latest alias rows for the tier 1 and 2 hosting providers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost-map): derive hosted alias targets from the cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(databricks): skip alias rows when checking cache-write rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../get_llm_provider_logic.py | 6 +- .../model_latest_aliases.py | 72 ++++- litellm/model_latest_aliases.json | 21 -- ...odel_prices_and_context_window_backup.json | 294 ++++++++++++++++++ model_prices_and_context_window.json | 294 ++++++++++++++++++ .../test_get_llm_provider_logic.py | 59 +++- .../test_model_latest_aliases.py | 69 +++- .../test_databricks_cost_calculator.py | 2 + .../test_moonshot_chat_transformation.py | 5 +- tests/unit/rust_bridge/test_detectors.py | 4 +- 10 files changed, 760 insertions(+), 66 deletions(-) delete mode 100644 litellm/model_latest_aliases.json diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index b2851345302..bbb1b19819f 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -194,11 +194,7 @@ def get_llm_provider( resolved_model, provider, dynamic_api_key, resolved_api_base = _get_llm_provider( model, custom_llm_provider, api_base, api_key, litellm_params ) - explicit_provider: Final = custom_llm_provider or (litellm_params.custom_llm_provider if litellm_params else None) - target: Final = ( - resolve_model_latest_alias(resolved_model, provider) if explicit_provider or "/" in model else resolved_model - ) - return target, provider, dynamic_api_key, resolved_api_base + return resolve_model_latest_alias(resolved_model, provider), provider, dynamic_api_key, resolved_api_base def _get_llm_provider( diff --git a/litellm/litellm_core_utils/model_latest_aliases.py b/litellm/litellm_core_utils/model_latest_aliases.py index e3b1a21dd5a..ee45ccc9b3c 100644 --- a/litellm/litellm_core_utils/model_latest_aliases.py +++ b/litellm/litellm_core_utils/model_latest_aliases.py @@ -1,33 +1,75 @@ from collections.abc import Mapping -from functools import cache -from importlib.resources import files from types import MappingProxyType -from typing import Final +from typing import Final, cast # noqa: TID251 # litellm.model_cost is an untyped module global -from pydantic import TypeAdapter - -_ALIASES_ADAPTER: Final = TypeAdapter(dict[str, dict[str, str]]) +import litellm +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry + + +def _cost_map() -> Mapping[str, object]: + return cast("Mapping[str, object]", litellm.model_cost) # cast-ok: per-lookup catalog validation is too slow + + +def _alias_entry(key: str) -> AliasedCostEntry | None: + entry: Final = _cost_map().get(key) + return entry if isinstance(entry, AliasedCostEntry) and entry.names[:1] not in ((), (key,)) else None + + +def _alias_provider(key: str, entry: AliasedCostEntry) -> str | None: + """The provider an alias is requested through: its key prefix for a hosted alias such as + ``bedrock/claude-haiku-latest`` (whose target row belongs to ``bedrock_converse``), else the target's provider""" + prefix, separator, _ = key.partition("/") + if separator and prefix in litellm.provider_list: + return prefix + provider: Final = entry.get("litellm_provider") + return provider if isinstance(provider, str) else None + + +def _alias_target(key: str, provider: str) -> str | None: + entry: Final = _alias_entry(key) + if entry is None or _alias_provider(key, entry) != provider: + return None + return entry.names[0].removeprefix(f"{provider}/") -@cache def get_model_latest_aliases() -> Mapping[str, Mapping[str, str]]: - aliases: Final = _ALIASES_ADAPTER.validate_json(files("litellm").joinpath("model_latest_aliases.json").read_bytes()) - return MappingProxyType({provider: MappingProxyType(models) for provider, models in aliases.items()}) + """``{provider: {alias: target}}`` for every alias row in the cost map, with provider prefixes stripped""" + rows: Final = tuple( + (provider, key, entry.names[0]) + for key in _cost_map() + if (entry := _alias_entry(key)) is not None and (provider := _alias_provider(key, entry)) is not None + ) + return MappingProxyType( + { + provider: MappingProxyType( + { + alias.removeprefix(f"{provider}/"): target.removeprefix(f"{provider}/") + for row_provider, alias, target in rows + if row_provider == provider + } + ) + for provider in dict.fromkeys(provider for provider, _, _ in rows) + } + ) def resolve_model_latest_alias(model: str, provider: str) -> str: - aliases: Final = get_model_latest_aliases().get(provider) - return aliases.get(model, model) if aliases is not None else model + return _alias_target(f"{provider}/{model}", provider) or _alias_target(model, provider) or model def expand_model_latest_alias(model: str) -> tuple[str, ...]: provider, separator, name = model.partition("/") - target: Final = resolve_model_latest_alias(name, provider) if separator else name - return (model, f"{provider}/{target}") if separator and target != name else (model,) + if not separator: + entry: Final = _alias_entry(model) + return (model,) if entry is None else (model, entry.names[0]) + target: Final = resolve_model_latest_alias(name, provider) + return (model, f"{provider}/{target}") if target != name else (model,) def is_model_latest_alias(model: str, custom_llm_provider: str | None = None) -> bool: prefix, separator, name = model.partition("/") - provider: Final = custom_llm_provider or prefix + provider: Final = custom_llm_provider or (prefix if separator else None) + if provider is None: + return _alias_entry(model) is not None candidate: Final = name if separator and prefix == provider else model - return bool(custom_llm_provider or separator) and resolve_model_latest_alias(candidate, provider) != candidate + return resolve_model_latest_alias(candidate, provider) != candidate diff --git a/litellm/model_latest_aliases.json b/litellm/model_latest_aliases.json deleted file mode 100644 index 7b0fa5d3d39..00000000000 --- a/litellm/model_latest_aliases.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "openai": { - "gpt-luna-latest": "gpt-6-luna", - "gpt-sol-latest": "gpt-6.1-sol", - "gpt-terra-latest": "gpt-6.1-terra", - "gpt-astra-latest": "gpt-6-astra", - "gpt-mini-latest": "gpt-5.4-mini" - }, - "anthropic": { - "claude-opus-latest": "claude-opus-5-5", - "claude-sonnet-latest": "claude-sonnet-5-5", - "claude-haiku-latest": "claude-haiku-5-5", - "claude-fable-latest": "claude-fable-5-1" - }, - "xai": {"grok-latest": "grok-4.7"}, - "moonshot": {"kimi-latest": "kimi-k3"}, - "zai": {"glm-latest": "glm-5.3", "glm-flash-latest": "glm-5.3-flash"}, - "deepseek": {"deepseek-pro-latest": "deepseek-v4-pro", "deepseek-flash-latest": "deepseek-v4-flash"}, - "dashscope": {"qwen-max-latest": "qwen3.8-max", "qwen-flash-latest": "qwen3.8-flash"}, - "minimax": {"minimax-latest": "MiniMax-M3"} -} diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ac5e99c9313..b3412518cd7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1969,6 +1969,9 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/claude-fable-latest": { + "alias_of": "global.anthropic.claude-fable-5-1" + }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, @@ -2305,6 +2308,9 @@ "region": "us-east-1" } }, + "bedrock/claude-opus-latest": { + "alias_of": "global.anthropic.claude-opus-5-5" + }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -4063,6 +4069,9 @@ "deprecation_date": "2027-12-05", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, + "azure_ai/claude-fable-latest": { + "alias_of": "azure_ai/claude-fable-5-1" + }, "azure_ai/claude-mythos-5-1": { "input_cost_per_token": 1e-05, "output_cost_per_token": 5e-05, @@ -4165,6 +4174,9 @@ "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, + "azure_ai/claude-opus-latest": { + "alias_of": "azure_ai/claude-opus-5-5" + }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -4382,6 +4394,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-astra-latest": { + "alias_of": "azure_ai/gpt-6-astra" + }, "azure_ai/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -4430,6 +4445,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-luna-latest": { + "alias_of": "azure_ai/gpt-6-luna" + }, "azure_ai/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -4526,6 +4544,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-sol-latest": { + "alias_of": "azure_ai/gpt-6.1-sol" + }, "azure_ai/gpt-6.1-terra": { "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, @@ -4574,6 +4595,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-terra-latest": { + "alias_of": "azure_ai/gpt-6.1-terra" + }, "azure_ai/gpt-5.5": { "deprecation_date": "2027-10-26", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -5036,6 +5060,9 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure_ai/gpt-mini-latest": { + "alias_of": "azure_ai/gpt-5.4-mini" + }, "azure_ai/gpt-5.4-mini-2026-03-17": { "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -9080,6 +9107,9 @@ "cache_read_input_token_cost_flex": 5e-07, "cache_creation_input_token_cost_flex": 6.25e-06 }, + "azure/gpt-astra-latest": { + "alias_of": "azure/gpt-6-astra" + }, "azure/gpt-6-astra-2026-09-03": { "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, @@ -9184,6 +9214,9 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, "output_cost_per_token_flex": 2.5e-07 }, + "azure/gpt-luna-latest": { + "alias_of": "azure/gpt-6-luna" + }, "azure/gpt-6-luna-2026-09-22": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -9403,6 +9436,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure/gpt-terra-latest": { + "alias_of": "azure/gpt-6.1-terra" + }, "azure/gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -9455,6 +9491,9 @@ "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, + "azure/gpt-sol-latest": { + "alias_of": "azure/gpt-6.1-sol" + }, "azure/gpt-6.1-sol-2026-09-29": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -11146,6 +11185,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, + "azure/gpt-mini-latest": { + "alias_of": "azure/gpt-5.4-mini" + }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -13212,6 +13254,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/kimi-latest": { + "alias_of": "azure_ai/FW-Kimi-K3" + }, "azure_ai/FW-MiniMax-M2.5": { "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.3e-08, @@ -13251,6 +13296,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/minimax-latest": { + "alias_of": "azure_ai/FW-MiniMax-M3" + }, "azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 6e-08, @@ -13656,6 +13704,9 @@ "cache_read_input_token_cost": 1.45e-07, "supports_prompt_caching": true }, + "azure_ai/deepseek-pro-latest": { + "alias_of": "azure_ai/deepseek-v4-pro" + }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -13674,6 +13725,9 @@ "cache_read_input_token_cost": 2.8e-08, "supports_prompt_caching": true }, + "azure_ai/deepseek-flash-latest": { + "alias_of": "azure_ai/deepseek-v4-flash" + }, "azure_ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", @@ -16623,6 +16677,9 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing", "supports_web_search": true }, + "claude-sonnet-latest": { + "alias_of": "claude-sonnet-5-5" + }, "claude-sonnet-4-6": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -17024,6 +17081,9 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-fable-latest": { + "alias_of": "claude-fable-5-1" + }, "claude-opus-5-5": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -17069,6 +17129,9 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-opus-latest": { + "alias_of": "claude-opus-5-5" + }, "claude-opus-5": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -18868,6 +18931,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "dashscope/qwen-max-latest": { + "alias_of": "dashscope/qwen3.8-max" + }, "dashscope/qwen3.8-flash": { "cache_creation_input_token_cost": 2e-07, "cache_read_input_token_cost": 1.6e-08, @@ -18887,6 +18953,9 @@ "supports_vision": true, "supports_web_search": true }, + "dashscope/qwen-flash-latest": { + "alias_of": "dashscope/qwen3.8-flash" + }, "dashscope/qwen3.8-omni-flash": { "cache_read_input_token_cost": 1.6e-08, "input_cost_per_token": 1.5e-07, @@ -21767,6 +21836,9 @@ "supports_vision": true, "thinking_always_on": true }, + "databricks/claude-fable-latest": { + "alias_of": "databricks/databricks-claude-fable-5-1" + }, "databricks/databricks-claude-haiku-4-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_creation_input_token_cost_above_1hr": 1.99997e-06, @@ -22007,6 +22079,9 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true }, + "databricks/claude-opus-latest": { + "alias_of": "databricks/databricks-claude-opus-5-5" + }, "databricks/databricks-claude-sonnet-4": { "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, @@ -22724,6 +22799,9 @@ "cache_read_input_token_cost_priority": 4.55e-07, "cache_creation_input_token_cost_priority": 2.45e-06 }, + "databricks/glm-latest": { + "alias_of": "databricks/databricks-glm-5-3" + }, "databricks/databricks-glm-5-3-flash": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 3.003e-08, @@ -22757,6 +22835,9 @@ "cache_read_input_token_cost_priority": 5.25e-08, "cache_creation_input_token_cost_priority": 2.625e-07 }, + "databricks/glm-flash-latest": { + "alias_of": "databricks/databricks-glm-5-3-flash" + }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.2502e-07, @@ -22919,6 +23000,9 @@ "cache_read_input_token_cost_priority": 1.5001e-07, "cache_creation_input_token_cost_priority": 1.50003e-06 }, + "databricks/gpt-mini-latest": { + "alias_of": "databricks/databricks-gpt-5-4-mini" + }, "databricks/databricks-gpt-5-4-nano": { "cache_creation_input_token_cost": 1.9999e-07, "cache_read_input_token_cost": 2.002e-08, @@ -23315,6 +23399,9 @@ "cache_read_input_token_cost_priority": 5.25e-07, "cache_creation_input_token_cost_priority": 5.25e-06 }, + "databricks/kimi-latest": { + "alias_of": "databricks/databricks-kimi-k3" + }, "databricks/system.ai.kimi-k3": { "litellm_provider": "databricks", "mode": "chat", @@ -28843,6 +28930,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/minimax-latest": { + "alias_of": "fireworks_ai/accounts/fireworks/models/minimax-m3" + }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -33789,6 +33879,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-fable-latest": { + "alias_of": "github_copilot/claude-fable-5.1" + }, "github_copilot/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, @@ -33832,6 +33925,9 @@ "supports_vision": true, "thinking_always_on": false }, + "github_copilot/claude-haiku-latest": { + "alias_of": "github_copilot/claude-haiku-5.5" + }, "github_copilot/claude-opus-4.8": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, @@ -33920,6 +34016,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-opus-latest": { + "alias_of": "github_copilot/claude-opus-5.5" + }, "github_copilot/claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, "cache_read_input_token_cost": 2e-07, @@ -33963,6 +34062,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-sonnet-latest": { + "alias_of": "github_copilot/claude-sonnet-5.5" + }, "github_copilot/gemini-3.7-flash": { "cache_read_input_token_cost": 7.5e-08, "deprecation_date": "2026-10-19", @@ -34308,6 +34410,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-astra-latest": { + "alias_of": "github_copilot/gpt-6-astra" + }, "github_copilot/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -34331,6 +34436,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-luna-latest": { + "alias_of": "github_copilot/gpt-6-luna" + }, "github_copilot/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -34377,6 +34485,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-sol-latest": { + "alias_of": "github_copilot/gpt-6.1-sol" + }, "github_copilot/kimi-k3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -34394,6 +34505,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/kimi-latest": { + "alias_of": "github_copilot/kimi-k3" + }, "github_copilot/mai-code-1.1-flash": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, @@ -37687,6 +37801,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-astra-latest": { + "alias_of": "gpt-6-astra" + }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, @@ -37884,6 +38001,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-luna-latest": { + "alias_of": "gpt-6-luna" + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_12hr": 1e-05, @@ -39067,6 +39187,9 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "gpt-mini-latest": { + "alias_of": "gpt-5.4-mini" + }, "gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, @@ -43025,6 +43148,9 @@ "max_input_tokens": 1000000, "max_output_tokens": 128000 }, + "minimax/minimax-latest": { + "alias_of": "minimax/MiniMax-M3" + }, "minimax/MiniMax-H3": { "litellm_provider": "minimax", "mode": "video_generation", @@ -44535,6 +44661,9 @@ "supports_video_input": true, "supports_vision": true }, + "moonshot/kimi-latest": { + "alias_of": "moonshot/kimi-k3" + }, "moonshot/moonshot-v1-128k": { "deprecation_date": "2026-08-31", "input_cost_per_token": 2e-06, @@ -50136,6 +50265,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "together_ai/deepseek-pro-latest": { + "alias_of": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" + }, "together_ai/meta-models/Muse-Glimmer-30B": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, @@ -50182,6 +50314,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/kimi-latest": { + "alias_of": "together_ai/moonshotai/Kimi-K3" + }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "deprecation_date": "2026-11-06", @@ -50232,6 +50367,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "together_ai/glm-latest": { + "alias_of": "together_ai/zai-org/GLM-5.3" + }, "together_ai/zai-org/GLM-5.3-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.5e-07, @@ -50249,6 +50387,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/glm-flash-latest": { + "alias_of": "together_ai/zai-org/GLM-5.3-Flash" + }, "tts-1": { "deprecation_date": "2027-01-06", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", @@ -53019,6 +53160,9 @@ "output_cost_per_token_batches": 2.5e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-fable-latest": { + "alias_of": "vertex_ai/claude-fable-5-1" + }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", "regional_endpoint_uplift_multiplier": 1.1, @@ -53217,6 +53361,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-opus-latest": { + "alias_of": "vertex_ai/claude-opus-5-5" + }, "vertex_ai/claude-opus-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, @@ -55765,6 +55912,9 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-latest": { + "alias_of": "xai/grok-4.7" + }, "zai.glm-4.7": { "deprecation_date": "2027-03-30", "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", @@ -55866,6 +56016,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "zai/glm-latest": { + "alias_of": "zai/glm-5.3" + }, "zai/glm-5.3-flash": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 3e-08, @@ -55883,6 +56036,9 @@ "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true }, + "zai/glm-flash-latest": { + "alias_of": "zai/glm-5.3-flash" + }, "zai/glm-5v-turbo": { "cache_read_input_token_cost": 2.4e-07, "input_cost_per_token": 1.2e-06, @@ -61519,6 +61675,9 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html", "supports_web_search": true }, + "bedrock_mantle/gpt-astra-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6-astra" + }, "bedrock_mantle/openai.gpt-6-sol": { "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, @@ -61599,6 +61758,9 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html", "supports_web_search": true }, + "bedrock_mantle/gpt-luna-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6-luna" + }, "us.openai.gpt-6-astra": { "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 1.1e-05, @@ -61767,6 +61929,9 @@ "/v1/responses" ] }, + "bedrock/gpt-astra-latest": { + "alias_of": "global.openai.gpt-6-astra" + }, "openai.gpt-6-sol": { "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, @@ -61909,6 +62074,9 @@ "/v1/responses" ] }, + "bedrock/gpt-luna-latest": { + "alias_of": "global.openai.gpt-6-luna" + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, @@ -62186,6 +62354,9 @@ "supports_forced_tool_use": false, "token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-east-1"} }, + "bedrock_mantle/claude-opus-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-opus-5-5" + }, "bedrock_mantle/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, @@ -62226,6 +62397,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock_mantle/claude-sonnet-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-sonnet-5-5" + }, "us.xai.grok-4.6": { "supports_regex_lookaround": false, "input_cost_per_token": 2.2e-06, @@ -63796,6 +63970,9 @@ "token_count_framing": 23, "token_count_tool_framing": 12 }, + "deepseek/deepseek-flash-latest": { + "alias_of": "deepseek/deepseek-v4-flash" + }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, @@ -63904,6 +64081,9 @@ "token_count_framing": 76, "token_count_tool_framing": 75 }, + "deepseek/deepseek-pro-latest": { + "alias_of": "deepseek/deepseek-v4-pro" + }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 1.45e-07, @@ -65327,6 +65507,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/kimi-latest": { + "alias_of": "fireworks_ai/accounts/fireworks/models/kimi-k3" + }, "fireworks_ai/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 7e-09, "cache_read_input_token_cost_priority": 8.75e-09, @@ -69000,6 +69183,9 @@ "output_cost_per_token": 4.7e-07, "source": "https://api.together.ai/v1/models" }, + "together_ai/qwen-flash-latest": { + "alias_of": "together_ai/Qwen/Qwen3.8-Flash" + }, "together_ai/moonshotai/Kimi-K2.5-fp4": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.8e-06, @@ -69216,6 +69402,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/glm-latest": { + "alias_of": "azure_ai/FW-GLM-5.3" + }, "azure_ai/FW-GLM-5.3-Flash": { "cache_read_input_token_cost": 3.8e-08, "input_cost_per_token": 1.88e-07, @@ -69231,6 +69420,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/glm-flash-latest": { + "alias_of": "azure_ai/FW-GLM-5.3-Flash" + }, "azure_ai/FW-GPT-OSS-120B": { "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 8.2e-08, @@ -71236,6 +71428,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/glm-latest": { + "alias_of": "baseten/zai-org/GLM-5.3" + }, "baseten/zai-org/GLM-5.3-Fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -74425,6 +74620,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/kimi-latest": { + "alias_of": "global.moonshotai.kimi-k3" + }, "us.moonshotai.kimi-k3": { "supports_regex_lookaround": false, "cache_creation_input_token_cost": 4.125e-06, @@ -74975,6 +75173,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/kimi-latest": { + "alias_of": "baseten/moonshotai/Kimi-K3" + }, "baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, @@ -75057,6 +75258,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/glm-flash-latest": { + "alias_of": "baseten/zai-org/GLM-5.3-Flash" + }, "baseten/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.3e-07, @@ -75073,6 +75277,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "baseten/deepseek-flash-latest": { + "alias_of": "baseten/deepseek-ai/DeepSeek-V4-Flash-0731" + }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, @@ -75106,6 +75313,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "baseten/deepseek-pro-latest": { + "alias_of": "baseten/deepseek-ai/DeepSeek-V4-Pro-0813" + }, "baseten/zai-org/GLM-5.2-Fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -76097,6 +76307,9 @@ "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, + "azure_ai/claude-sonnet-latest": { + "alias_of": "azure_ai/claude-sonnet-5-5" + }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "bedrock_output_config_effort_ceiling": "xhigh", @@ -76261,6 +76474,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock/claude-sonnet-latest": { + "alias_of": "global.anthropic.claude-sonnet-5-5" + }, "jp.anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, @@ -76520,6 +76736,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-sonnet-latest": { + "alias_of": "vertex_ai/claude-sonnet-5-5" + }, "vertex_ai/claude-sonnet-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, @@ -76632,6 +76851,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/grok-latest": { + "alias_of": "global.xai.grok-4.7" + }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, "supported_endpoints": ["/v1/chat/completions"], @@ -76804,6 +77026,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-sol-latest": { + "alias_of": "gpt-6.1-sol" + }, "global.openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -76850,6 +77075,9 @@ "supports_sampling_params": false, "supports_xhigh_reasoning_effort": true }, + "bedrock/gpt-sol-latest": { + "alias_of": "global.openai.gpt-6.1-sol" + }, "bedrock_mantle/openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, @@ -76898,6 +77126,9 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "bedrock_mantle/gpt-sol-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6.1-sol" + }, "us.openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, @@ -76989,6 +77220,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/grok-latest": { + "alias_of": "vertex_ai/xai/grok-4.7" + }, "azure_ai/kimi-k2-thinking": { "deprecation_date": "2026-03-29", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -77046,6 +77280,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "bedrock/glm-latest": { + "alias_of": "global.zai.glm-5.3" + }, "us.zai.glm-5.3": { "cache_creation_input_token_cost": 2.31e-06, "cache_creation_input_token_cost_batches": 1.155e-06, @@ -77269,6 +77506,9 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/grok-latest": { + "alias_of": "azure_ai/grok-4.7" + }, "in.moonshotai.kimi-k3": { "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_flex": 2.0625e-06, @@ -77355,6 +77595,9 @@ "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, "cache_read_input_token_cost_above_100k_tokens": 5e-08 }, + "claude-haiku-latest": { + "alias_of": "claude-haiku-5-5" + }, "bedrock_mantle/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, @@ -77401,6 +77644,9 @@ "prompt_cache_min_tokens": 512, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html" }, + "bedrock_mantle/claude-haiku-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-haiku-5-5" + }, "bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 6e-07, "output_cost_per_token_above_100k_tokens": 3e-06, @@ -77632,6 +77878,9 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide" }, + "azure_ai/claude-haiku-latest": { + "alias_of": "azure_ai/claude-haiku-5-5" + }, "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 6e-07, "output_cost_per_token_above_100k_tokens": 3e-06, @@ -77822,6 +78071,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock/claude-haiku-latest": { + "alias_of": "global.anthropic.claude-haiku-5-5" + }, "jp.anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, @@ -78029,6 +78281,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, + "vertex_ai/claude-haiku-latest": { + "alias_of": "vertex_ai/claude-haiku-5-5" + }, "vertex_ai/claude-haiku-5-5@default": { "input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, @@ -78933,6 +79188,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/grok-latest": { + "alias_of": "github_copilot/grok-4.7" + }, "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 6e-09, "input_cost_per_token": 3e-07, @@ -80356,6 +80614,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-terra-latest": { + "alias_of": "gpt-6.1-terra" + }, "chatgpt/gpt-6.1-terra": { "litellm_provider": "chatgpt", "max_input_tokens": 922000, @@ -80414,6 +80675,9 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/gpt-terra-latest": { + "alias_of": "global.openai.gpt-6.1-terra" + }, "us.openai.gpt-6.1-terra": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, @@ -80493,6 +80757,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-terra-latest": { + "alias_of": "github_copilot/gpt-6.1-terra" + }, "refuseless/glm-5.3-abliterated": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1e-06, @@ -80647,6 +80914,9 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "bedrock_mantle/gpt-terra-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6.1-terra" + }, "clf_ai_gateway/glm-5.3": { "metadata": { "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." @@ -82196,6 +82466,9 @@ "supports_vision": true, "thinking_always_on": false }, + "databricks/claude-haiku-latest": { + "alias_of": "databricks/databricks-claude-haiku-5-5" + }, "databricks/databricks-claude-sonnet-5-5": { "cache_creation_input_token_cost": 2.49998e-06, "cache_creation_input_token_cost_above_1hr": 4.00001e-06, @@ -82225,6 +82498,9 @@ "supports_vision": false, "thinking_always_on": true }, + "databricks/claude-sonnet-latest": { + "alias_of": "databricks/databricks-claude-sonnet-5-5" + }, "databricks/databricks-deepseek-v4-1-flash": { "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.003e-08, @@ -82457,6 +82733,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, + "databricks/gpt-sol-latest": { + "alias_of": "databricks/databricks-gpt-6-1-sol" + }, "databricks/databricks-gpt-6-1-terra": { "cache_creation_input_token_cost": 1.24999e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.49998e-06, @@ -82497,6 +82776,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06 }, + "databricks/gpt-terra-latest": { + "alias_of": "databricks/databricks-gpt-6-1-terra" + }, "databricks/databricks-gpt-6-astra": { "cache_creation_input_token_cost": 1.249997e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, @@ -82537,6 +82819,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05 }, + "databricks/gpt-astra-latest": { + "alias_of": "databricks/databricks-gpt-6-astra" + }, "databricks/databricks-gpt-6-luna": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.4997e-07, @@ -82577,6 +82862,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07 }, + "databricks/gpt-luna-latest": { + "alias_of": "databricks/databricks-gpt-6-luna" + }, "databricks/databricks-gpt-6-sol": { "cache_creation_input_token_cost": 2.49998e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.00003e-06, @@ -82707,6 +82995,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "databricks/grok-latest": { + "alias_of": "databricks/databricks-grok-4-7" + }, "databricks/databricks-kimi-k2-7": { "cache_creation_input_token_cost": 9.4997e-07, "cache_read_input_token_cost": 1.8998e-07, @@ -86639,6 +86930,9 @@ "supports_vision": true, "use_openai_responses_path": true }, + "bedrock_mantle/grok-latest": { + "alias_of": "bedrock_mantle/xai.grok-4.7" + }, "together_ai/ricursive-ai/Ripple-65B-Preview": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ac5e99c9313..b3412518cd7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1969,6 +1969,9 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/claude-fable-latest": { + "alias_of": "global.anthropic.claude-fable-5-1" + }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, @@ -2305,6 +2308,9 @@ "region": "us-east-1" } }, + "bedrock/claude-opus-latest": { + "alias_of": "global.anthropic.claude-opus-5-5" + }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -4063,6 +4069,9 @@ "deprecation_date": "2027-12-05", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, + "azure_ai/claude-fable-latest": { + "alias_of": "azure_ai/claude-fable-5-1" + }, "azure_ai/claude-mythos-5-1": { "input_cost_per_token": 1e-05, "output_cost_per_token": 5e-05, @@ -4165,6 +4174,9 @@ "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, + "azure_ai/claude-opus-latest": { + "alias_of": "azure_ai/claude-opus-5-5" + }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -4382,6 +4394,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-astra-latest": { + "alias_of": "azure_ai/gpt-6-astra" + }, "azure_ai/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -4430,6 +4445,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-luna-latest": { + "alias_of": "azure_ai/gpt-6-luna" + }, "azure_ai/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -4526,6 +4544,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-sol-latest": { + "alias_of": "azure_ai/gpt-6.1-sol" + }, "azure_ai/gpt-6.1-terra": { "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, @@ -4574,6 +4595,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-terra-latest": { + "alias_of": "azure_ai/gpt-6.1-terra" + }, "azure_ai/gpt-5.5": { "deprecation_date": "2027-10-26", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -5036,6 +5060,9 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure_ai/gpt-mini-latest": { + "alias_of": "azure_ai/gpt-5.4-mini" + }, "azure_ai/gpt-5.4-mini-2026-03-17": { "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -9080,6 +9107,9 @@ "cache_read_input_token_cost_flex": 5e-07, "cache_creation_input_token_cost_flex": 6.25e-06 }, + "azure/gpt-astra-latest": { + "alias_of": "azure/gpt-6-astra" + }, "azure/gpt-6-astra-2026-09-03": { "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, @@ -9184,6 +9214,9 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, "output_cost_per_token_flex": 2.5e-07 }, + "azure/gpt-luna-latest": { + "alias_of": "azure/gpt-6-luna" + }, "azure/gpt-6-luna-2026-09-22": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -9403,6 +9436,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure/gpt-terra-latest": { + "alias_of": "azure/gpt-6.1-terra" + }, "azure/gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -9455,6 +9491,9 @@ "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, + "azure/gpt-sol-latest": { + "alias_of": "azure/gpt-6.1-sol" + }, "azure/gpt-6.1-sol-2026-09-29": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -11146,6 +11185,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, + "azure/gpt-mini-latest": { + "alias_of": "azure/gpt-5.4-mini" + }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -13212,6 +13254,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/kimi-latest": { + "alias_of": "azure_ai/FW-Kimi-K3" + }, "azure_ai/FW-MiniMax-M2.5": { "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.3e-08, @@ -13251,6 +13296,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/minimax-latest": { + "alias_of": "azure_ai/FW-MiniMax-M3" + }, "azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 6e-08, @@ -13656,6 +13704,9 @@ "cache_read_input_token_cost": 1.45e-07, "supports_prompt_caching": true }, + "azure_ai/deepseek-pro-latest": { + "alias_of": "azure_ai/deepseek-v4-pro" + }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -13674,6 +13725,9 @@ "cache_read_input_token_cost": 2.8e-08, "supports_prompt_caching": true }, + "azure_ai/deepseek-flash-latest": { + "alias_of": "azure_ai/deepseek-v4-flash" + }, "azure_ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", @@ -16623,6 +16677,9 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing", "supports_web_search": true }, + "claude-sonnet-latest": { + "alias_of": "claude-sonnet-5-5" + }, "claude-sonnet-4-6": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -17024,6 +17081,9 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-fable-latest": { + "alias_of": "claude-fable-5-1" + }, "claude-opus-5-5": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -17069,6 +17129,9 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-opus-latest": { + "alias_of": "claude-opus-5-5" + }, "claude-opus-5": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -18868,6 +18931,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "dashscope/qwen-max-latest": { + "alias_of": "dashscope/qwen3.8-max" + }, "dashscope/qwen3.8-flash": { "cache_creation_input_token_cost": 2e-07, "cache_read_input_token_cost": 1.6e-08, @@ -18887,6 +18953,9 @@ "supports_vision": true, "supports_web_search": true }, + "dashscope/qwen-flash-latest": { + "alias_of": "dashscope/qwen3.8-flash" + }, "dashscope/qwen3.8-omni-flash": { "cache_read_input_token_cost": 1.6e-08, "input_cost_per_token": 1.5e-07, @@ -21767,6 +21836,9 @@ "supports_vision": true, "thinking_always_on": true }, + "databricks/claude-fable-latest": { + "alias_of": "databricks/databricks-claude-fable-5-1" + }, "databricks/databricks-claude-haiku-4-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_creation_input_token_cost_above_1hr": 1.99997e-06, @@ -22007,6 +22079,9 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true }, + "databricks/claude-opus-latest": { + "alias_of": "databricks/databricks-claude-opus-5-5" + }, "databricks/databricks-claude-sonnet-4": { "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, @@ -22724,6 +22799,9 @@ "cache_read_input_token_cost_priority": 4.55e-07, "cache_creation_input_token_cost_priority": 2.45e-06 }, + "databricks/glm-latest": { + "alias_of": "databricks/databricks-glm-5-3" + }, "databricks/databricks-glm-5-3-flash": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 3.003e-08, @@ -22757,6 +22835,9 @@ "cache_read_input_token_cost_priority": 5.25e-08, "cache_creation_input_token_cost_priority": 2.625e-07 }, + "databricks/glm-flash-latest": { + "alias_of": "databricks/databricks-glm-5-3-flash" + }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.2502e-07, @@ -22919,6 +23000,9 @@ "cache_read_input_token_cost_priority": 1.5001e-07, "cache_creation_input_token_cost_priority": 1.50003e-06 }, + "databricks/gpt-mini-latest": { + "alias_of": "databricks/databricks-gpt-5-4-mini" + }, "databricks/databricks-gpt-5-4-nano": { "cache_creation_input_token_cost": 1.9999e-07, "cache_read_input_token_cost": 2.002e-08, @@ -23315,6 +23399,9 @@ "cache_read_input_token_cost_priority": 5.25e-07, "cache_creation_input_token_cost_priority": 5.25e-06 }, + "databricks/kimi-latest": { + "alias_of": "databricks/databricks-kimi-k3" + }, "databricks/system.ai.kimi-k3": { "litellm_provider": "databricks", "mode": "chat", @@ -28843,6 +28930,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/minimax-latest": { + "alias_of": "fireworks_ai/accounts/fireworks/models/minimax-m3" + }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -33789,6 +33879,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-fable-latest": { + "alias_of": "github_copilot/claude-fable-5.1" + }, "github_copilot/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, @@ -33832,6 +33925,9 @@ "supports_vision": true, "thinking_always_on": false }, + "github_copilot/claude-haiku-latest": { + "alias_of": "github_copilot/claude-haiku-5.5" + }, "github_copilot/claude-opus-4.8": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, @@ -33920,6 +34016,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-opus-latest": { + "alias_of": "github_copilot/claude-opus-5.5" + }, "github_copilot/claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, "cache_read_input_token_cost": 2e-07, @@ -33963,6 +34062,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-sonnet-latest": { + "alias_of": "github_copilot/claude-sonnet-5.5" + }, "github_copilot/gemini-3.7-flash": { "cache_read_input_token_cost": 7.5e-08, "deprecation_date": "2026-10-19", @@ -34308,6 +34410,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-astra-latest": { + "alias_of": "github_copilot/gpt-6-astra" + }, "github_copilot/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -34331,6 +34436,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-luna-latest": { + "alias_of": "github_copilot/gpt-6-luna" + }, "github_copilot/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -34377,6 +34485,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-sol-latest": { + "alias_of": "github_copilot/gpt-6.1-sol" + }, "github_copilot/kimi-k3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -34394,6 +34505,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/kimi-latest": { + "alias_of": "github_copilot/kimi-k3" + }, "github_copilot/mai-code-1.1-flash": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, @@ -37687,6 +37801,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-astra-latest": { + "alias_of": "gpt-6-astra" + }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, @@ -37884,6 +38001,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-luna-latest": { + "alias_of": "gpt-6-luna" + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_12hr": 1e-05, @@ -39067,6 +39187,9 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "gpt-mini-latest": { + "alias_of": "gpt-5.4-mini" + }, "gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, @@ -43025,6 +43148,9 @@ "max_input_tokens": 1000000, "max_output_tokens": 128000 }, + "minimax/minimax-latest": { + "alias_of": "minimax/MiniMax-M3" + }, "minimax/MiniMax-H3": { "litellm_provider": "minimax", "mode": "video_generation", @@ -44535,6 +44661,9 @@ "supports_video_input": true, "supports_vision": true }, + "moonshot/kimi-latest": { + "alias_of": "moonshot/kimi-k3" + }, "moonshot/moonshot-v1-128k": { "deprecation_date": "2026-08-31", "input_cost_per_token": 2e-06, @@ -50136,6 +50265,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "together_ai/deepseek-pro-latest": { + "alias_of": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" + }, "together_ai/meta-models/Muse-Glimmer-30B": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, @@ -50182,6 +50314,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/kimi-latest": { + "alias_of": "together_ai/moonshotai/Kimi-K3" + }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "deprecation_date": "2026-11-06", @@ -50232,6 +50367,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "together_ai/glm-latest": { + "alias_of": "together_ai/zai-org/GLM-5.3" + }, "together_ai/zai-org/GLM-5.3-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.5e-07, @@ -50249,6 +50387,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/glm-flash-latest": { + "alias_of": "together_ai/zai-org/GLM-5.3-Flash" + }, "tts-1": { "deprecation_date": "2027-01-06", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", @@ -53019,6 +53160,9 @@ "output_cost_per_token_batches": 2.5e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-fable-latest": { + "alias_of": "vertex_ai/claude-fable-5-1" + }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", "regional_endpoint_uplift_multiplier": 1.1, @@ -53217,6 +53361,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-opus-latest": { + "alias_of": "vertex_ai/claude-opus-5-5" + }, "vertex_ai/claude-opus-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, @@ -55765,6 +55912,9 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-latest": { + "alias_of": "xai/grok-4.7" + }, "zai.glm-4.7": { "deprecation_date": "2027-03-30", "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", @@ -55866,6 +56016,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "zai/glm-latest": { + "alias_of": "zai/glm-5.3" + }, "zai/glm-5.3-flash": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 3e-08, @@ -55883,6 +56036,9 @@ "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true }, + "zai/glm-flash-latest": { + "alias_of": "zai/glm-5.3-flash" + }, "zai/glm-5v-turbo": { "cache_read_input_token_cost": 2.4e-07, "input_cost_per_token": 1.2e-06, @@ -61519,6 +61675,9 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html", "supports_web_search": true }, + "bedrock_mantle/gpt-astra-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6-astra" + }, "bedrock_mantle/openai.gpt-6-sol": { "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, @@ -61599,6 +61758,9 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html", "supports_web_search": true }, + "bedrock_mantle/gpt-luna-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6-luna" + }, "us.openai.gpt-6-astra": { "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 1.1e-05, @@ -61767,6 +61929,9 @@ "/v1/responses" ] }, + "bedrock/gpt-astra-latest": { + "alias_of": "global.openai.gpt-6-astra" + }, "openai.gpt-6-sol": { "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, @@ -61909,6 +62074,9 @@ "/v1/responses" ] }, + "bedrock/gpt-luna-latest": { + "alias_of": "global.openai.gpt-6-luna" + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, @@ -62186,6 +62354,9 @@ "supports_forced_tool_use": false, "token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-east-1"} }, + "bedrock_mantle/claude-opus-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-opus-5-5" + }, "bedrock_mantle/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, @@ -62226,6 +62397,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock_mantle/claude-sonnet-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-sonnet-5-5" + }, "us.xai.grok-4.6": { "supports_regex_lookaround": false, "input_cost_per_token": 2.2e-06, @@ -63796,6 +63970,9 @@ "token_count_framing": 23, "token_count_tool_framing": 12 }, + "deepseek/deepseek-flash-latest": { + "alias_of": "deepseek/deepseek-v4-flash" + }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, @@ -63904,6 +64081,9 @@ "token_count_framing": 76, "token_count_tool_framing": 75 }, + "deepseek/deepseek-pro-latest": { + "alias_of": "deepseek/deepseek-v4-pro" + }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 1.45e-07, @@ -65327,6 +65507,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/kimi-latest": { + "alias_of": "fireworks_ai/accounts/fireworks/models/kimi-k3" + }, "fireworks_ai/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 7e-09, "cache_read_input_token_cost_priority": 8.75e-09, @@ -69000,6 +69183,9 @@ "output_cost_per_token": 4.7e-07, "source": "https://api.together.ai/v1/models" }, + "together_ai/qwen-flash-latest": { + "alias_of": "together_ai/Qwen/Qwen3.8-Flash" + }, "together_ai/moonshotai/Kimi-K2.5-fp4": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.8e-06, @@ -69216,6 +69402,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/glm-latest": { + "alias_of": "azure_ai/FW-GLM-5.3" + }, "azure_ai/FW-GLM-5.3-Flash": { "cache_read_input_token_cost": 3.8e-08, "input_cost_per_token": 1.88e-07, @@ -69231,6 +69420,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/glm-flash-latest": { + "alias_of": "azure_ai/FW-GLM-5.3-Flash" + }, "azure_ai/FW-GPT-OSS-120B": { "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 8.2e-08, @@ -71236,6 +71428,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/glm-latest": { + "alias_of": "baseten/zai-org/GLM-5.3" + }, "baseten/zai-org/GLM-5.3-Fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -74425,6 +74620,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/kimi-latest": { + "alias_of": "global.moonshotai.kimi-k3" + }, "us.moonshotai.kimi-k3": { "supports_regex_lookaround": false, "cache_creation_input_token_cost": 4.125e-06, @@ -74975,6 +75173,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/kimi-latest": { + "alias_of": "baseten/moonshotai/Kimi-K3" + }, "baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, @@ -75057,6 +75258,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/glm-flash-latest": { + "alias_of": "baseten/zai-org/GLM-5.3-Flash" + }, "baseten/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.3e-07, @@ -75073,6 +75277,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "baseten/deepseek-flash-latest": { + "alias_of": "baseten/deepseek-ai/DeepSeek-V4-Flash-0731" + }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, @@ -75106,6 +75313,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "baseten/deepseek-pro-latest": { + "alias_of": "baseten/deepseek-ai/DeepSeek-V4-Pro-0813" + }, "baseten/zai-org/GLM-5.2-Fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -76097,6 +76307,9 @@ "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, + "azure_ai/claude-sonnet-latest": { + "alias_of": "azure_ai/claude-sonnet-5-5" + }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "bedrock_output_config_effort_ceiling": "xhigh", @@ -76261,6 +76474,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock/claude-sonnet-latest": { + "alias_of": "global.anthropic.claude-sonnet-5-5" + }, "jp.anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, @@ -76520,6 +76736,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-sonnet-latest": { + "alias_of": "vertex_ai/claude-sonnet-5-5" + }, "vertex_ai/claude-sonnet-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, @@ -76632,6 +76851,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/grok-latest": { + "alias_of": "global.xai.grok-4.7" + }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, "supported_endpoints": ["/v1/chat/completions"], @@ -76804,6 +77026,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-sol-latest": { + "alias_of": "gpt-6.1-sol" + }, "global.openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -76850,6 +77075,9 @@ "supports_sampling_params": false, "supports_xhigh_reasoning_effort": true }, + "bedrock/gpt-sol-latest": { + "alias_of": "global.openai.gpt-6.1-sol" + }, "bedrock_mantle/openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, @@ -76898,6 +77126,9 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "bedrock_mantle/gpt-sol-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6.1-sol" + }, "us.openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, @@ -76989,6 +77220,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/grok-latest": { + "alias_of": "vertex_ai/xai/grok-4.7" + }, "azure_ai/kimi-k2-thinking": { "deprecation_date": "2026-03-29", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -77046,6 +77280,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "bedrock/glm-latest": { + "alias_of": "global.zai.glm-5.3" + }, "us.zai.glm-5.3": { "cache_creation_input_token_cost": 2.31e-06, "cache_creation_input_token_cost_batches": 1.155e-06, @@ -77269,6 +77506,9 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/grok-latest": { + "alias_of": "azure_ai/grok-4.7" + }, "in.moonshotai.kimi-k3": { "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_flex": 2.0625e-06, @@ -77355,6 +77595,9 @@ "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, "cache_read_input_token_cost_above_100k_tokens": 5e-08 }, + "claude-haiku-latest": { + "alias_of": "claude-haiku-5-5" + }, "bedrock_mantle/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, @@ -77401,6 +77644,9 @@ "prompt_cache_min_tokens": 512, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html" }, + "bedrock_mantle/claude-haiku-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-haiku-5-5" + }, "bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 6e-07, "output_cost_per_token_above_100k_tokens": 3e-06, @@ -77632,6 +77878,9 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide" }, + "azure_ai/claude-haiku-latest": { + "alias_of": "azure_ai/claude-haiku-5-5" + }, "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 6e-07, "output_cost_per_token_above_100k_tokens": 3e-06, @@ -77822,6 +78071,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock/claude-haiku-latest": { + "alias_of": "global.anthropic.claude-haiku-5-5" + }, "jp.anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, @@ -78029,6 +78281,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, + "vertex_ai/claude-haiku-latest": { + "alias_of": "vertex_ai/claude-haiku-5-5" + }, "vertex_ai/claude-haiku-5-5@default": { "input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, @@ -78933,6 +79188,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/grok-latest": { + "alias_of": "github_copilot/grok-4.7" + }, "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 6e-09, "input_cost_per_token": 3e-07, @@ -80356,6 +80614,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-terra-latest": { + "alias_of": "gpt-6.1-terra" + }, "chatgpt/gpt-6.1-terra": { "litellm_provider": "chatgpt", "max_input_tokens": 922000, @@ -80414,6 +80675,9 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/gpt-terra-latest": { + "alias_of": "global.openai.gpt-6.1-terra" + }, "us.openai.gpt-6.1-terra": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, @@ -80493,6 +80757,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-terra-latest": { + "alias_of": "github_copilot/gpt-6.1-terra" + }, "refuseless/glm-5.3-abliterated": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1e-06, @@ -80647,6 +80914,9 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "bedrock_mantle/gpt-terra-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6.1-terra" + }, "clf_ai_gateway/glm-5.3": { "metadata": { "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." @@ -82196,6 +82466,9 @@ "supports_vision": true, "thinking_always_on": false }, + "databricks/claude-haiku-latest": { + "alias_of": "databricks/databricks-claude-haiku-5-5" + }, "databricks/databricks-claude-sonnet-5-5": { "cache_creation_input_token_cost": 2.49998e-06, "cache_creation_input_token_cost_above_1hr": 4.00001e-06, @@ -82225,6 +82498,9 @@ "supports_vision": false, "thinking_always_on": true }, + "databricks/claude-sonnet-latest": { + "alias_of": "databricks/databricks-claude-sonnet-5-5" + }, "databricks/databricks-deepseek-v4-1-flash": { "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.003e-08, @@ -82457,6 +82733,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, + "databricks/gpt-sol-latest": { + "alias_of": "databricks/databricks-gpt-6-1-sol" + }, "databricks/databricks-gpt-6-1-terra": { "cache_creation_input_token_cost": 1.24999e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.49998e-06, @@ -82497,6 +82776,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06 }, + "databricks/gpt-terra-latest": { + "alias_of": "databricks/databricks-gpt-6-1-terra" + }, "databricks/databricks-gpt-6-astra": { "cache_creation_input_token_cost": 1.249997e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, @@ -82537,6 +82819,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05 }, + "databricks/gpt-astra-latest": { + "alias_of": "databricks/databricks-gpt-6-astra" + }, "databricks/databricks-gpt-6-luna": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.4997e-07, @@ -82577,6 +82862,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07 }, + "databricks/gpt-luna-latest": { + "alias_of": "databricks/databricks-gpt-6-luna" + }, "databricks/databricks-gpt-6-sol": { "cache_creation_input_token_cost": 2.49998e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.00003e-06, @@ -82707,6 +82995,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "databricks/grok-latest": { + "alias_of": "databricks/databricks-grok-4-7" + }, "databricks/databricks-kimi-k2-7": { "cache_creation_input_token_cost": 9.4997e-07, "cache_read_input_token_cost": 1.8998e-07, @@ -86639,6 +86930,9 @@ "supports_vision": true, "use_openai_responses_path": true }, + "bedrock_mantle/grok-latest": { + "alias_of": "bedrock_mantle/xai.grok-4.7" + }, "together_ai/ricursive-ai/Ripple-65B-Preview": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index de252b17f00..b4e3ef2c566 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -30,9 +30,9 @@ CUSTOM_PROVIDER: Final = "test-onprem-llm" @pytest.mark.parametrize( ("model", "provider"), - (("openai/gpt-luna-latest", None), ("gpt-luna-latest", "openai")), + (("openai/gpt-luna-latest", None), ("gpt-luna-latest", "openai"), ("gpt-luna-latest", None)), ) -def test_latest_alias_resolves_to_the_bundled_target(model: str, provider: str | None) -> None: +def test_latest_alias_resolves_to_the_cost_map_target(model: str, provider: str | None) -> None: target: Final = get_model_latest_aliases()["openai"]["gpt-luna-latest"] resolved, resolved_provider, _, api_base = get_llm_provider( model, custom_llm_provider=provider, api_base="https://api.openai.com/v1" @@ -42,7 +42,12 @@ def test_latest_alias_resolves_to_the_bundled_target(model: str, provider: str | @pytest.mark.parametrize( "model", - ("gemini/gemini-pro-latest", "gemini/gemini-flash-latest", "dashscope/qwen-plus-latest", "mistral/mistral-large-latest"), + ( + "gemini/gemini-pro-latest", + "gemini/gemini-flash-latest", + "dashscope/qwen-plus-latest", + "mistral/mistral-large-latest", + ), ) def test_provider_native_latest_alias_passes_through(model: str) -> None: provider, name = model.split("/", 1) @@ -153,6 +158,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -205,6 +211,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -225,6 +232,7 @@ _SCALAR_DEFAULTS = { "api_key": getattr(litellm, "api_key", None), } + @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -247,12 +255,14 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider(): _, response, _, _ = litellm.get_llm_provider(model="anthropic.claude-v2:1") assert response == "bedrock" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_fireworks(): # tests finetuned fireworks models - https://github.com/BerriAI/litellm/issues/4923 model, custom_llm_provider, _, _ = litellm.get_llm_provider(model="fireworks_ai/accounts/my-test-1234") @@ -260,17 +270,20 @@ def test_get_llm_provider_fireworks(): # tests finetuned fireworks models - htt assert custom_llm_provider == "fireworks_ai" assert model == "accounts/my-test-1234" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_catch_all(): _, response, _, _ = litellm.get_llm_provider(model="*") assert response == "openai" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_gpt_instruct(): _, response, _, _ = litellm.get_llm_provider(model="gpt-3.5-turbo-instruct-0914") assert response == "text-completion-openai" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_mistral_custom_api_base(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -281,6 +294,7 @@ def test_get_llm_provider_mistral_custom_api_base(): assert model == "mistral-large-fr" assert api_base == "https://mistral-large-fr-ishaan.francecentral.inference.ai.azure.com/v1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_deepseek_custom_api_base(): os.environ["DEEPSEEK_API_BASE"] = "MY-FAKE-BASE" @@ -293,6 +307,7 @@ def test_get_llm_provider_deepseek_custom_api_base(): os.environ.pop("DEEPSEEK_API_BASE") + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_vertex_ai_image_models(monkeypatch): monkeypatch.setattr(litellm, "vertex_ai_image_models", set()) @@ -310,6 +325,7 @@ def test_get_llm_provider_vertex_ai_image_models(monkeypatch): ) assert custom_llm_provider == "vertex_ai" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_ai21_chat(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -319,6 +335,7 @@ def test_get_llm_provider_ai21_chat(): assert model == "jamba-1.5-large" assert api_base == "https://api.ai21.com/studio/v1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_ai21_chat_test2(): """ @@ -335,6 +352,7 @@ def test_get_llm_provider_ai21_chat_test2(): assert model == "jamba-1.5-large" assert api_base == "https://api.ai21.com/studio/v1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_cohere_chat_test2(): """ @@ -350,6 +368,7 @@ def test_get_llm_provider_cohere_chat_test2(): assert custom_llm_provider == "cohere_chat" assert model == "command-r-plus-08-2024" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_azure_o1(): @@ -359,6 +378,7 @@ def test_get_llm_provider_azure_o1(): assert custom_llm_provider == "azure" assert model == "o1-mini" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_hosted_vllm_default_api_key(): from litellm.litellm_core_utils.get_llm_provider_logic import ( @@ -373,6 +393,7 @@ def test_hosted_vllm_default_api_key(): ) assert dynamic_api_key == "fake-api-key" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_jina_ai(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -382,6 +403,7 @@ def test_get_llm_provider_jina_ai(): assert api_base == "https://api.jina.ai/v1" assert model == "jina-embeddings-v3" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_hosted_vllm(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -391,6 +413,7 @@ def test_get_llm_provider_hosted_vllm(): assert model == "llama-3.1-70b-instruct" assert dynamic_api_key == "fake-api-key" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_llamafile(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -401,6 +424,7 @@ def test_get_llm_provider_llamafile(): assert dynamic_api_key == "fake-api-key" assert api_base == "http://127.0.0.1:8080/v1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_watson_text(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -409,6 +433,7 @@ def test_get_llm_provider_watson_text(): assert custom_llm_provider == "watsonx_text" assert model == "watson-text-to-speech" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_azure_global_standard_get_llm_provider(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -418,6 +443,7 @@ def test_azure_global_standard_get_llm_provider(): ) assert custom_llm_provider == "azure_ai" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_nova_bedrock_converse(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -426,6 +452,7 @@ def test_nova_bedrock_converse(): assert custom_llm_provider == "bedrock" assert model == "amazon.nova-micro-v1:0" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_bedrock_invoke_anthropic(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -434,6 +461,7 @@ def test_bedrock_invoke_anthropic(): assert custom_llm_provider == "bedrock" assert model == "invoke/anthropic.claude-haiku-4-5-20251001-v1:0" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize("model", ["xai/grok-2-vision-latest", "grok-2-vision-latest"]) def test_xai_api_base(model): @@ -450,6 +478,7 @@ def test_xai_api_base(model): assert api_base == "https://api.x.ai/v1" assert dynamic_api_key == "xai-my-specialkey" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_litellm_proxy_custom_llm_provider(): """ @@ -479,6 +508,7 @@ def test_get_litellm_proxy_custom_llm_provider(): assert key == expected_api_key assert base == expected_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_litellm_proxy_with_args_override_env_vars(): """ @@ -510,6 +540,7 @@ def test_get_litellm_proxy_with_args_override_env_vars(): assert key == arg_api_key assert base == arg_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_litellm_proxy_model_prefix_stripping(): """ @@ -540,6 +571,7 @@ def test_get_litellm_proxy_model_prefix_stripping(): assert key == expected_api_key assert base == expected_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true(): """ @@ -568,6 +600,7 @@ def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true(): assert key == proxy_api_key assert base == proxy_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true_model_prefix(): """ @@ -594,6 +627,7 @@ def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true_model_prefix(): assert key == proxy_api_key assert base == proxy_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_use_proxy_arg_true(): """ @@ -623,6 +657,7 @@ def test_get_llm_provider_use_proxy_arg_true(): assert key == proxy_api_key assert base == proxy_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_use_proxy_arg_true_with_direct_args(): """ @@ -659,6 +694,7 @@ def test_get_llm_provider_use_proxy_arg_true_with_direct_args(): assert key == arg_api_key # Should use the argument key assert base == arg_api_base # Should use the argument base + @pytest.fixture def shipped_generalizations(): """Install the rules shipped in the bundled backup, then restore. @@ -682,6 +718,7 @@ def shipped_generalizations(): finally: set_fallback_generalizations(previous) + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") class TestClaudeModelPatternMatching: """ @@ -749,9 +786,7 @@ class TestClaudeModelPatternMatching: @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") class TestTypesafeJevPatternMatching: - def test_every_priced_jev_id_routes_to_typesafe( - self, shipped_generalizations: None - ) -> None: + def test_every_priced_jev_id_routes_to_typesafe(self, shipped_generalizations: None) -> None: from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap priced_ids: Final = tuple( @@ -760,9 +795,9 @@ class TestTypesafeJevPatternMatching: if key.startswith("typesafe/") ) assert "jev-latest" in priced_ids - assert { - model: litellm.get_llm_provider(model=model)[1] for model in priced_ids - } == dict.fromkeys(priced_ids, "typesafe") + assert {model: litellm.get_llm_provider(model=model)[1] for model in priced_ids} == dict.fromkeys( + priced_ids, "typesafe" + ) @pytest.mark.parametrize( "model", @@ -837,7 +872,9 @@ def test_metadata_callers_never_log_in_for_a_declared_chatgpt_deployment( assert get_api_base("chatgpt/gpt-5.4", {"model": "chatgpt/gpt-5.4"}) == "https://chatgpt.com/backend-api/codex" identity = _resolve_model("chatgpt/gpt-5.4", None) assert identity is not None and identity.provider == "chatgpt" - group = Router(model_list=[{"model_name": "codex", "litellm_params": {"model": "chatgpt/gpt-5.4"}}]).registry().group( - "codex" + group = ( + Router(model_list=[{"model_name": "codex", "litellm_params": {"model": "chatgpt/gpt-5.4"}}]) + .registry() + .group("codex") ) assert group is not None and group.providers == ["chatgpt"] diff --git a/tests/unit/litellm_core_utils/test_model_latest_aliases.py b/tests/unit/litellm_core_utils/test_model_latest_aliases.py index c052a24b4dc..c6def915640 100644 --- a/tests/unit/litellm_core_utils/test_model_latest_aliases.py +++ b/tests/unit/litellm_core_utils/test_model_latest_aliases.py @@ -3,14 +3,17 @@ from typing import Final import pytest import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.litellm_core_utils.model_latest_aliases import ( expand_model_latest_alias, get_model_latest_aliases, is_model_latest_alias, + resolve_model_latest_alias, ) -@pytest.mark.parametrize("model", ("", "gpt-luna-latest", "openai/unknown", "gemini/gemini-pro-latest")) +@pytest.mark.parametrize("model", ("", "kimi-latest", "openai/unknown", "gemini/gemini-pro-latest")) def test_non_alias_expansion_keeps_the_requested_name(model: str) -> None: assert expand_model_latest_alias(model) == (model,) @@ -21,15 +24,56 @@ def test_alias_expansion_keeps_requested_and_target_names() -> None: assert expand_model_latest_alias(f"{provider}/{alias}") == (f"{provider}/{alias}", f"{provider}/{target}") -def test_latest_alias_targets_have_chat_pricing_without_shadowing_native_models() -> None: - for provider, aliases in get_model_latest_aliases().items(): - for alias, target in aliases.items(): +def test_bare_alias_expands_to_its_cost_map_owner() -> None: + alias, target = next(iter(get_model_latest_aliases()["openai"].items())) + assert expand_model_latest_alias(alias) == (alias, target) + + +def test_latest_aliases_price_and_route_like_their_targets() -> None: + aliases: Final = get_model_latest_aliases() + assert aliases + for provider, provider_aliases in aliases.items(): + for alias, target in provider_aliases.items(): + prefixed: Final = f"{provider}/{alias}" + alias_key: Final = prefixed if prefixed in litellm.model_cost else alias target_key: Final = f"{provider}/{target}" if f"{provider}/{target}" in litellm.model_cost else target - entry: Final = litellm.model_cost[target_key] - assert (entry["litellm_provider"], entry["mode"]) == (provider, "chat") - assert f"{provider}/{alias}" not in litellm.model_cost - assert litellm.model_cost.get(alias, {}).get("litellm_provider") != provider - assert litellm.get_llm_provider(f"{provider}/{alias}")[:2] == (target, provider) + entry: Final = litellm.model_cost[alias_key] + assert entry["mode"] in ("chat", "responses") + assert entry == litellm.model_cost[target_key] + assert resolve_model_latest_alias(alias, provider) == target + if provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: + continue + assert litellm.get_llm_provider(prefixed)[:2] == (target, provider) + assert litellm.get_llm_provider(alias, custom_llm_provider=provider)[:2] == (target, provider) + + +@pytest.mark.parametrize( + ("model", "provider", "target_provider"), + ( + ("bedrock/claude-haiku-latest", "bedrock", "bedrock_converse"), + ("vertex_ai/claude-haiku-latest", "vertex_ai", "vertex_ai-anthropic_models"), + ("azure/gpt-mini-latest", "azure", "azure"), + ("fireworks_ai/kimi-latest", "fireworks_ai", "fireworks_ai"), + ), +) +def test_hosted_alias_is_sent_as_the_hosts_model_id(model: str, provider: str, target_provider: str) -> None: + entry: Final = litellm.model_cost[model] + assert isinstance(entry, AliasedCostEntry) + name: Final = model.partition("/")[2] + target: Final = entry.names[0].removeprefix(f"{provider}/") + assert target != name + assert entry["litellm_provider"] == target_provider + assert litellm.get_llm_provider(model)[:2] == (target, provider) + assert is_model_latest_alias(model) + assert resolve_model_latest_alias(name, provider) == target + assert get_model_latest_aliases()[provider][name] == target + + +def test_alias_resolution_is_scoped_to_the_owning_provider() -> None: + alias, target = next(iter(get_model_latest_aliases()["openai"].items())) + assert resolve_model_latest_alias(alias, "openai") == target + assert resolve_model_latest_alias(alias, "anthropic") == alias + assert resolve_model_latest_alias(target, "openai") == target @pytest.mark.parametrize( @@ -37,11 +81,14 @@ def test_latest_alias_targets_have_chat_pricing_without_shadowing_native_models( ( ("openai/gpt-luna-latest", None, True), ("gpt-luna-latest", "openai", True), - ("gpt-luna-latest", None, False), + ("gpt-luna-latest", None, True), + ("gpt-luna-latest", "anthropic", False), + ("moonshot/kimi-latest", None, True), + ("kimi-latest", None, False), ("gemini/gemini-pro-latest", None, False), ("", None, False), ("openai/unknown-model", None, False), ), ) -def test_latest_alias_detection_requires_an_explicit_provider(model: str, provider: str | None, expected: bool) -> None: +def test_latest_alias_detection_follows_the_cost_map_owner(model: str, provider: str | None, expected: bool) -> None: assert is_model_latest_alias(model, provider) is expected diff --git a/tests/unit/llms/databricks/test_databricks_cost_calculator.py b/tests/unit/llms/databricks/test_databricks_cost_calculator.py index c35aa6e4cf0..e9bc75119aa 100644 --- a/tests/unit/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/unit/llms/databricks/test_databricks_cost_calculator.py @@ -6,6 +6,7 @@ from typing import Final import pytest import litellm +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.llms.databricks.cost_calculator import cost_per_token from litellm.types.utils import ModelInfo, Usage @@ -213,6 +214,7 @@ def test_every_model_without_published_cache_dbu_bills_cache_writes_at_its_own_i if model.startswith("databricks/") and info.get("input_cost_per_token") and model not in PUBLISHED_DBU_PER_MILLION + and not (isinstance(info, AliasedCostEntry) and info.names[0] != model) ] for model in without_published_rates: diff --git a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py index b054dd84c22..2f58330e427 100644 --- a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py @@ -13,7 +13,7 @@ import pytest import litellm import litellm.utils -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, get_model_cost_map from litellm.litellm_core_utils.model_latest_aliases import get_model_latest_aliases from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig @@ -715,7 +715,8 @@ class TestMoonshotReasoningEffort: @pytest.fixture(autouse=True) def force_local_model_cost(self, monkeypatch): - monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) @pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6"]) def test_reasoning_model_supports_reasoning_effort(self, model): diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 3001d23e57e..dfa29e24b2e 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -51,8 +51,10 @@ def call(**kwargs: object) -> NativeCall: ("openai/gpt-luna-latest", None, True), ("gpt-luna-latest", "openai", True), ("gemini/gemini-pro-latest", None, False), - ("gpt-luna-latest", None, False), + ("gpt-luna-latest", None, True), ("anthropic/claude-opus-latest", None, True), + ("moonshot/kimi-latest", None, True), + ("kimi-latest", None, False), ("claude-opus-latest", "anthropic", True), ("", None, False), ), From 5bc25ff8ea5d169463e83b2b4efd17d44131bf78 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 12:01:36 -0800 Subject: [PATCH 10/17] fix(cost-map): update Together Qwen3.7-Max pricing and add Rnj-1 Instruct (#50548) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 13 ++++++++++--- model_prices_and_context_window.json | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b3412518cd7..6b704577303 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -50176,13 +50176,13 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 7.5e-06, "source": "https://www.together.ai/pricing", "supports_prompt_caching": true }, @@ -71774,6 +71774,13 @@ "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" }, + "together_ai/essentialai/rnj-1-instruct": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://www.together.ai/models/rnj-1-instruct" + }, "together_ai/google/gemma-4-31B-it": { "deprecation_date": "2026-09-15", "deprecation_source": "https://docs.together.ai/docs/deprecations", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b3412518cd7..6b704577303 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -50176,13 +50176,13 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 7.5e-06, "source": "https://www.together.ai/pricing", "supports_prompt_caching": true }, @@ -71774,6 +71774,13 @@ "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" }, + "together_ai/essentialai/rnj-1-instruct": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://www.together.ai/models/rnj-1-instruct" + }, "together_ai/google/gemma-4-31B-it": { "deprecation_date": "2026-09-15", "deprecation_source": "https://docs.together.ai/docs/deprecations", From 8d9ddb349e5e72ee12f0ef2ff3daad22e4d6c9ab Mon Sep 17 00:00:00 2001 From: Yogeshwaran Ravichandran <96047771+yogeshwaran10@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:59:12 +0530 Subject: [PATCH 11/17] feat(guardrails): add prebuilt credential patterns for OpenAI, Anthropic, and Gemini (#50268) * feat(guardrails): add prebuilt credential patterns for JWT, and OpenAI, Anthropic, Gemini API keys * fix(guardrails): prevent quadratic backtracking in jwt_token pattern * test(guardrails): move credential pattern tests into test_patterns.py * feat(guardrails): remove jwt_token to keep credential patterns strictly bounded * fix(guardrails): mask whole provider keys and keep openai_api_key to OpenAI's key shapes End each pattern so a key ending in "-" is masked in full, require OpenAI's T3BlbkFJ marker on bare sk- keys so other vendors' sk- keys and LiteLLM virtual keys are left alone, add sk-None- user keys, and replace the registration-only tests with whole-match, length-boundary, and negative cases * fix(guardrails): require the T3BlbkFJ marker on every OpenAI key form and match keys after escaped separators openai_api_key now needs the marker on the sk-proj-, sk-svcacct-, sk-admin- and sk-None- forms too, as gitleaks and TruffleHog do. anthropic_api_key and gemini_api_key also start a match right after a literal \n or \t or a percent-encoded byte, so keys in pasted JSON strings and URL-encoded logs are masked. * fix(guardrails): bound the OpenAI key regex, match Gemini prefixes case-sensitively, and hold back long keys in streamed replies * fix(guardrails): size the streaming holdback per pattern, stop OpenAI key tails before the next key, and allow non-ASCII text after a key * fix(guardrails): mask content filter pattern matches found in the original text so overlapping patterns cannot split a key * fix(guardrails): let a block pattern see the text after earlier masking patterns, as before Masking patterns still match the original text and merge overlapping matches into one tag, but a BLOCK pattern checks the text with the masking patterns listed before it applied, so a request main masked and passed is not blocked now. Detections name only the patterns whose tag was written. * fix(guardrails): keep content filter masks in order and hold back the longest OpenAI key Masking patterns run in configured order on the masked text as on main, so a contextual pattern still matches next to an earlier tag, and each also matches the original text so an earlier tag cannot split a key. Detections follow the configured pattern, not its name, and the OpenAI streaming holdback covers the pattern's 419-character longest match. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_content_filter/content_filter.py | 157 ++++++++++++---- .../litellm_content_filter/patterns.json | 24 ++- .../litellm_content_filter/patterns.py | 13 +- .../content_filter/test_content_filter.py | 142 ++++++++++++++- .../content_filter/test_patterns.py | 168 ++++++++++++++++++ 5 files changed, 468 insertions(+), 36 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index b1e3d51bbc1..6f077650a92 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -6,15 +6,17 @@ to detect and block/mask sensitive content. """ import asyncio +import bisect import contextvars import itertools import json import os import re import time -from collections.abc import AsyncGenerator, AsyncIterable, Callable, Coroutine, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, Callable, Coroutine, Iterable, Mapping, Sequence from dataclasses import dataclass, replace from datetime import datetime +from functools import partial from re import Pattern from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, cast @@ -79,7 +81,7 @@ from .competitor_intent import ( AirlineCompetitorIntentChecker, BaseCompetitorIntentChecker, ) -from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern +from .patterns import PATTERN_EXTRA_CONFIG, PATTERN_STREAMING_HOLDBACK_CHARS, get_compiled_pattern MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1 _DECODED_ARGUMENTS: Final = TypeAdapter(object) @@ -223,10 +225,70 @@ async def _stream_then_end_of_stream_chunk( @dataclass(frozen=True, slots=True) class _StreamedScanPlan: context_chars: int + holdback_chars: int exception_phrases: tuple[str, ...] conditional_words: tuple[str, ...] +@dataclass(frozen=True, slots=True) +class _RedactionSpan: + start: int + end: int + pattern_index: int + pattern_name: str + + +def _merge_redaction_spans(spans: Iterable[_RedactionSpan]) -> tuple[_RedactionSpan, ...]: + ordered: Final = sorted(spans, key=lambda span: (span.start, -span.end)) + reach: Final = tuple(itertools.accumulate((span.end for span in ordered), max)) + first_indices: Final = tuple( + index for index, span in enumerate(ordered) if index == 0 or span.start >= reach[index - 1] + ) + last_indices: Final = (*(index - 1 for index in first_indices[1:]), len(ordered) - 1) + return tuple(replace(ordered[first], end=reach[last]) for first, last in zip(first_indices, last_indices)) + + +@dataclass(frozen=True, slots=True) +class _RedactedText: + spans: tuple[_RedactionSpan, ...] + tags: tuple[str, ...] + tag_starts: tuple[int, ...] + + def render(self, text: str) -> str: + previous_ends: Final = (0, *(span.end for span in self.spans)) + kept_parts: Final = ( + text[previous_end : span.start] + tag + for previous_end, span, tag in zip(previous_ends, self.spans, self.tags) + ) + return "".join((*kept_parts, text[previous_ends[-1] :])) + + def original_start(self, position: int) -> int: + index: Final = bisect.bisect_right(self.tag_starts, position) - 1 + if index < 0: + return position + tag_end: Final = self.tag_starts[index] + len(self.tags[index]) + if position < tag_end: + return self.spans[index].start + return position - tag_end + self.spans[index].end + + def original_end(self, position: int) -> int: + index: Final = bisect.bisect_left(self.tag_starts, position) - 1 + if index < 0: + return position + tag_end: Final = self.tag_starts[index] + len(self.tags[index]) + if position <= tag_end: + return self.spans[index].end + return position - tag_end + self.spans[index].end + + +def _redacted_text(spans: tuple[_RedactionSpan, ...], redaction_format: str) -> _RedactedText: + tags: Final = tuple(redaction_format.format(pattern_name=span.pattern_name.upper()) for span in spans) + shifts: Final = itertools.accumulate( + (len(tag) - (span.end - span.start) for span, tag in zip(spans, tags)), initial=0 + ) + return _RedactedText(spans, tags, tuple(span.start + shift for span, shift in zip(spans, shifts))) + + class CategoryFileData(TypedDict, total=False): category_name: str description: str @@ -987,21 +1049,6 @@ class ContentFilterGuardrail(CustomGuardrail): merged.append((start, end)) return merged - def _mask_spans(self, text: str, spans: list[tuple[int, int]], redaction: str) -> str: - """Apply masking for the provided spans using the given redaction tag.""" - - if not spans: - return text - - result_parts: Final[list[str]] = [] - previous_end = 0 - for start, end in spans: - result_parts.append(text[previous_end:start]) - result_parts.append(redaction) - previous_end = end - result_parts.append(text[previous_end:]) - return "".join(result_parts) - def _convert_word_number_sequence(self, sequence: str) -> str | None: """Convert a spelled-out digit sequence (e.g., 'One-Two') into digits.""" @@ -1392,10 +1439,8 @@ class ContentFilterGuardrail(CustomGuardrail): self, pattern_name: str, action: ContentFilterAction, - text: str, - spans: list[tuple[int, int]], detections: list[ContentFilterDetection] | None, - ) -> str: + ) -> None: """Handle regex pattern match detection and action.""" if detections is not None: pattern_detection: Final[PatternDetection] = { @@ -1413,11 +1458,49 @@ class ContentFilterGuardrail(CustomGuardrail): detail={"error": error_msg, "pattern": pattern_name}, ) elif action == ContentFilterAction.MASK: - redaction_tag: Final = self.pattern_redaction_format.format(pattern_name=pattern_name.upper()) - text = self._mask_spans(text, spans, redaction_tag) verbose_proxy_logger.info("Masked all %s matches in content", pattern_name) - return text + def _mask_step( + self, + text: str, + spans: tuple[_RedactionSpan, ...], + indexed_entry: tuple[int, CompiledPatternEntry], + ) -> tuple[_RedactionSpan, ...]: + index, pattern_entry = indexed_entry + if pattern_entry["action"] != ContentFilterAction.MASK: + return spans + original_matches: Final = self._find_pattern_spans(text, pattern_entry) + redacted_matches: Final = self._redacted_text_matches(text, spans, pattern_entry) if spans else () + return _merge_redaction_spans( + ( + *spans, + *( + _RedactionSpan(start, end, index, pattern_entry["pattern_name"]) + for start, end in (*original_matches, *redacted_matches) + ), + ) + ) + + def _redacted_text_matches( + self, text: str, spans: tuple[_RedactionSpan, ...], pattern_entry: CompiledPatternEntry + ) -> tuple[tuple[int, int], ...]: + redacted: Final = _redacted_text(spans, self.pattern_redaction_format) + return tuple( + (redacted.original_start(start), redacted.original_end(end)) + for start, end in self._find_pattern_spans(redacted.render(text), pattern_entry) + ) + + def _pattern_detected( + self, + text: str, + index: int, + pattern_entry: CompiledPatternEntry, + span_states: Sequence[tuple[_RedactionSpan, ...]], + ) -> bool: + if pattern_entry["action"] == ContentFilterAction.MASK: + return any(span.pattern_index == index for span in span_states[-1]) + text_after_earlier_masks: Final = _redacted_text(span_states[index], self.pattern_redaction_format).render(text) + return bool(self._find_pattern_spans(text_after_earlier_masks, pattern_entry)) def _handle_blocked_word_match( self, @@ -1506,13 +1589,14 @@ class ContentFilterGuardrail(CustomGuardrail): keyword, category_name, severity, action = category_keyword_match text = self._handle_category_keyword_match(keyword, category_name, severity, action, text, detections) - # Check regex patterns - process ALL patterns, not just first match - for pattern_entry in self.compiled_patterns: - spans = self._find_pattern_spans(text, pattern_entry) - if spans: - pattern_name = pattern_entry["pattern_name"] - action = pattern_entry["action"] - text = self._handle_pattern_match(pattern_name, action, text, spans, detections) + no_spans: Final[tuple[_RedactionSpan, ...]] = () + span_states: Final = tuple( + itertools.accumulate(enumerate(self.compiled_patterns), partial(self._mask_step, text), initial=no_spans) + ) + for index, pattern_entry in enumerate(self.compiled_patterns): + if self._pattern_detected(text, index, pattern_entry, span_states): + self._handle_pattern_match(pattern_entry["pattern_name"], pattern_entry["action"], detections) + text = _redacted_text(span_states[-1], self.pattern_redaction_format).render(text) # Check blocked words - iterate through ALL blocked words text_lower = text.lower() @@ -2071,7 +2155,9 @@ class ContentFilterGuardrail(CustomGuardrail): def _streamed_scan_plan(self) -> _StreamedScanPlan: """ Per-stream inputs for buffer trimming: the retained tail length (the default - context, widened to the longest configured keyword), the category exception + context, widened to the longest configured keyword), the unreleased tail length + (widened to the longest holdback a configured prebuilt pattern declares, since + provider keys run longer than the default), the category exception phrases, which suppress matches anywhere in the scanned text, and the conditional category words, which only match when paired inside one sentence. """ @@ -2081,6 +2167,13 @@ class ContentFilterGuardrail(CustomGuardrail): ) return _StreamedScanPlan( context_chars=max(CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS, longest_keyword), + holdback_chars=max( + ( + PATTERN_STREAMING_HOLDBACK_CHARS.get(entry["pattern_name"], CONTENT_FILTER_STREAMING_HOLDBACK_CHARS) + for entry in self.compiled_patterns + ), + default=CONTENT_FILTER_STREAMING_HOLDBACK_CHARS, + ), exception_phrases=tuple( phrase for category in self.loaded_categories.values() for phrase in category.exceptions ), @@ -2221,7 +2314,7 @@ class ContentFilterGuardrail(CustomGuardrail): safe_to_yield_len = max( previous_state.yielded_masked_text_len, - len(masked_text) - (0 if is_final else CONTENT_FILTER_STREAMING_HOLDBACK_CHARS), + len(masked_text) - (0 if is_final else plan.holdback_chars), ) released_text = masked_text[previous_state.yielded_masked_text_len : safe_to_yield_len] if isinstance(choice, TextChoices): diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json index db6dd36726c..dcb2dc5ce77 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json @@ -98,6 +98,28 @@ "category": "Credential Patterns", "description": "Detects generic API key patterns" }, + { + "name": "openai_api_key", + "display_name": "OpenAI API Key", + "pattern": "sk-(?:(?:proj|svcacct|admin|None)-[A-Za-z0-9_\\-]{20,200}T3BlbkFJ(?:(?!sk-(?:proj|svcacct|admin|None)-)[A-Za-z0-9_\\-]){20,200}|[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}(?![A-Za-z0-9_]))", + "streaming_holdback_chars": 419, + "category": "Credential Patterns", + "description": "Detects OpenAI API keys (sk-proj-..., sk-svcacct-..., sk-admin-..., sk-None-... and legacy sk-... keys, all carrying the T3BlbkFJ marker)" + }, + { + "name": "anthropic_api_key", + "display_name": "Anthropic API Key", + "pattern": "(?:(? str: + return "".join(await cls._collect_streamed_deltas(guardrail, chunks, metadata)) + + @staticmethod + async def _collect_streamed_deltas( + guardrail: ContentFilterGuardrail, + chunks: list[str], + metadata: dict[str, list[StandardLoggingGuardrailInformation]], + ) -> list[str]: from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices async def mock_stream(): @@ -951,7 +1042,7 @@ class TestContentFilterGuardrail: request_data={"messages": [], "model": "gpt-4o", "metadata": metadata}, ): yielded.append(chunk.choices[0].delta.content or "") - return "".join(yielded) + return yielded @pytest.mark.asyncio async def test_streaming_hook_scans_bounded_window_per_chunk(self): @@ -1068,6 +1159,53 @@ class TestContentFilterGuardrail: assert entry["guardrail_status"] == "guardrail_intervened" assert [d["keyword"] for d in entry["guardrail_response"]] == [phrase] + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("pattern_name", "key"), + [ + ("openai_api_key", "sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74), + ("openai_api_key", "sk-svcacct-" + "a" * 200 + "T3BlbkFJ" + "b" * 200), + ("anthropic_api_key", "sk-ant-api03-" + "a" * 93 + "AA"), + ("gemini_api_key", "AQ." + "a" * 55), + ], + ) + async def test_streaming_hook_masks_whole_provider_key_split_across_chunks(self, pattern_name: str, key: str): + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-provider-key", + patterns=[ + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name=pattern_name, + action=ContentFilterAction.MASK, + ) + ], + event_hook=GuardrailEventHooks.post_call, + ) + text: Final = f"Use {key} to call the API." + chunks: Final = [text[i : i + 4] for i in range(0, len(text), 4)] + + streamed: Final = await self._collect_streamed_text(guardrail, chunks, {}) + + assert streamed == f"Use [{pattern_name.upper()}_REDACTED] to call the API." + + @pytest.mark.asyncio + async def test_streaming_hook_keeps_default_holdback_without_provider_key_patterns(self): + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-older-credential-patterns", + patterns=[ + ContentFilterPattern(pattern_type="prebuilt", pattern_name=name, action=ContentFilterAction.MASK) + for name in ("aws_access_key", "aws_secret_key", "github_token", "slack_token", "generic_api_key") + ], + event_hook=GuardrailEventHooks.post_call, + ) + text: Final = "plain words with nothing secret in them. " * 12 + chunks: Final = [text[i : i + 10] for i in range(0, len(text), 10)] + + deltas: Final = await self._collect_streamed_deltas(guardrail, chunks, {}) + + assert "".join(deltas) == text + assert len(next(delta for delta in reversed(deltas) if delta)) <= CONTENT_FILTER_STREAMING_HOLDBACK_CHARS + @pytest.mark.asyncio async def test_streaming_hook_blocks_keyword_longer_than_scan_context(self): """ diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py b/tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py index d702b9e0116..3c71e814fe1 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py @@ -4,6 +4,8 @@ Tests for content filter pattern loading from JSON import json import os +import time +from typing import Final import pytest @@ -183,3 +185,169 @@ def test_eu_patterns_have_category(): assert ( pattern_name in eu_category_patterns ), f"Pattern {pattern_name} not in EU PII Patterns category" + + +def test_provider_key_patterns_are_offered_in_the_ui_as_credential_patterns() -> None: + ui_categories: Final = {pattern["name"]: pattern["category"] for pattern in get_pattern_metadata()} + assert [ui_categories.get(name) for name in ("openai_api_key", "anthropic_api_key", "gemini_api_key")] == [ + "Credential Patterns" + ] * 3 + + +@pytest.mark.parametrize( + ("pattern_name", "key"), + [ + pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-legacy"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74, id="openai-project"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-project-shortest"), + pytest.param("openai_api_key", "sk-svcacct-" + "a_b-" * 15 + "T3BlbkFJ" + "c-d_" * 15, id="openai-service-account"), + pytest.param("openai_api_key", "sk-admin-" + "a" * 58 + "T3BlbkFJ" + "b" * 58, id="openai-admin"), + pytest.param("openai_api_key", "sk-None-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-user-key"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 19 + "-", id="openai-trailing-hyphen"), + pytest.param( + "anthropic_api_key", "sk-ant-api03-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOP", id="anthropic-versioned" + ), + pytest.param( + "anthropic_api_key", "sk-ant-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTU", id="anthropic-unversioned" + ), + pytest.param("anthropic_api_key", "sk-ant-" + "a" * 20, id="anthropic-shortest"), + pytest.param("anthropic_api_key", "sk-ant-api03-" + "a_b-" * 23 + "AA", id="anthropic-urlsafe"), + pytest.param( + "anthropic_api_key", "sk-ant-api03-abcdefghijklmnopqrstuvwxyz0123456789-", id="anthropic-trailing-hyphen" + ), + pytest.param("gemini_api_key", "AIzaSyD-9tSrke72I6_AQp2h8HI4-uE6QNKQV0", id="gemini-legacy"), + pytest.param("gemini_api_key", "AIza" + "a" * 33, id="gemini-legacy-shortest"), + pytest.param("gemini_api_key", "AIza" + "a" * 36, id="gemini-legacy-longest"), + pytest.param("gemini_api_key", "AIza" + "a" * 34 + "-", id="gemini-legacy-trailing-hyphen"), + pytest.param("gemini_api_key", "AQ.Ab8RN6I-7xK9pL2mQ0vW5yZ1uX3tS4rE8qD7cB6aN5mP4oI3uY2", id="gemini-auth-key"), + pytest.param("gemini_api_key", "AQ." + "a" * 45, id="gemini-auth-key-shortest"), + pytest.param("gemini_api_key", "AQ." + "a" * 55, id="gemini-auth-key-longest"), + pytest.param("gemini_api_key", "AQ." + "a" * 49 + "-", id="gemini-auth-key-trailing-hyphen"), + ], +) +def test_credential_pattern_matches_whole_key(pattern_name: str, key: str) -> None: + match: Final = get_compiled_pattern(pattern_name).search(f"I keep getting 401 with {key}, is it expired?") + assert match is not None + assert match.group(0) == key + + +@pytest.mark.parametrize("separator", ["\\n", "\\t", "\\u003c", "Bearer%20", "?key%3D", "?key%253D", "GOOGLE_KEY_"]) +@pytest.mark.parametrize( + ("pattern_name", "key"), + [ + pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai"), + pytest.param("anthropic_api_key", "sk-ant-api03-" + "a" * 20, id="anthropic"), + pytest.param("gemini_api_key", "AIza" + "a" * 35, id="gemini-legacy"), + pytest.param("gemini_api_key", "AQ." + "a" * 50, id="gemini-auth-key"), + ], +) +def test_credential_pattern_matches_key_after_escaped_separator(pattern_name: str, key: str, separator: str) -> None: + match: Final = get_compiled_pattern(pattern_name).search(f'{{"log": "retrying with{separator}{key} failed"}}') + assert match is not None + assert match.group(0) == key + + +@pytest.mark.parametrize( + ("pattern_name", "text"), + [ + pytest.param("openai_api_key", "sk-short", id="openai-too-short"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 19, id="openai-project-too-short"), + pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 21, id="openai-legacy-too-long"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 48, id="openai-project-without-marker"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 19 + "T3BlbkFJ" + "b" * 20, id="openai-project-marker-too-early"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 19, id="openai-project-marker-too-late"), + pytest.param("openai_api_key", "SK-ADMIN-dashboard-redesign-phase-two", id="openai-prefixed-slug"), + pytest.param("openai_api_key", "desk-proj-management-dashboard-v2", id="openai-inside-longer-word"), + pytest.param( + "openai_api_key", "sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRST", id="openai-legacy-without-marker" + ), + pytest.param("openai_api_key", "sk-0123456789abcdef0123456789abcdef", id="openai-other-vendor-sk-key"), + pytest.param("openai_api_key", "sk-Xq3vR9tLm2Pa8KdW5yZgQw", id="openai-litellm-virtual-key"), + pytest.param( + "openai_api_key", + "sk-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCd-_x", + id="openai-litellm-virtual-key-raised-length", + ), + pytest.param("openai_api_key", "not-a-key", id="openai-no-prefix"), + pytest.param( + "openai_api_key", "sk-ant-api03-thisisanthropicnotopenaiabc123", id="openai-ignores-anthropic-key" + ), + pytest.param("openai_api_key", "ordinary sentence without any api key in it", id="openai-plain-text"), + pytest.param("anthropic_api_key", "sk-ant-" + "a" * 19, id="anthropic-too-short"), + pytest.param("anthropic_api_key", "flask-ant-design-system-components", id="anthropic-inside-longer-word"), + pytest.param( + "anthropic_api_key", + "sk-proj-abcdefghijklmnopqrstuvwxyzABCDEF1234567890", + id="anthropic-ignores-openai-project-key", + ), + pytest.param( + "anthropic_api_key", + "sk-admin-abcdefghijklmnopqrstuvwxyzABCDEF123456", + id="anthropic-ignores-openai-admin-key", + ), + pytest.param("anthropic_api_key", "sk-ant", id="anthropic-prefix-only"), + pytest.param("anthropic_api_key", "plain text with nothing sensitive", id="anthropic-plain-text"), + pytest.param("gemini_api_key", "AIza", id="gemini-prefix-only"), + pytest.param("gemini_api_key", "AIza" + "a" * 32, id="gemini-legacy-too-short"), + pytest.param("gemini_api_key", "AIza" + "a" * 37, id="gemini-legacy-too-long"), + pytest.param("gemini_api_key", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", id="gemini-wrong-prefix"), + pytest.param("gemini_api_key", "AQ.", id="gemini-auth-prefix-only"), + pytest.param("gemini_api_key", "AQ." + "a" * 44, id="gemini-auth-key-too-short"), + pytest.param("gemini_api_key", "AQ." + "a" * 56, id="gemini-auth-key-too-long"), + pytest.param("gemini_api_key", "AQ-" + "a" * 50, id="gemini-auth-key-wrong-separator"), + pytest.param("gemini_api_key", "x" + "AIza" + "a" * 35, id="gemini-inside-longer-token"), + pytest.param("gemini_api_key", "The AQ. was reported at 42 ppm today.", id="gemini-aq-acronym"), + pytest.param("gemini_api_key", "Is the question AQ. Maybe it is BQ.", id="gemini-aq-sentence-end"), + pytest.param("gemini_api_key", "plain text with no google key", id="gemini-plain-text"), + pytest.param( + "gemini_api_key", + "https://example.com/blog/aizawa-takeshi-interview-2024-tokyo-01", + id="gemini-lowercase-legacy-prefix", + ), + pytest.param( + "gemini_api_key", + "ssh aq.internal-observability-metrics-collector-eu-west-01", + id="gemini-lowercase-auth-prefix", + ), + ], +) +def test_credential_pattern_ignores_non_keys(pattern_name: str, text: str) -> None: + assert get_compiled_pattern(pattern_name).search(text) is None + + +@pytest.mark.parametrize("suffix", ["请检查", "é", "ü der Schlüssel"]) +@pytest.mark.parametrize( + ("pattern_name", "key"), + [ + pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-legacy"), + pytest.param("gemini_api_key", "AIza" + "a" * 35, id="gemini-legacy"), + pytest.param("gemini_api_key", "AQ." + "a" * 50, id="gemini-auth-key"), + ], +) +def test_credential_pattern_matches_key_followed_by_non_ascii_letters(pattern_name: str, key: str, suffix: str) -> None: + match: Final = get_compiled_pattern(pattern_name).search(f"我的密钥是{key}{suffix}") + assert match is not None + assert match.group(0) == key + + +@pytest.mark.parametrize( + "leading", + [ + pytest.param("sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74, id="another-key"), + pytest.param("sk-proj-" + "x" * 20 + "T3BlbkFJ" + "y" * 192, id="key-shaped-filler"), + pytest.param("sk-proj-" + "x" * 100 + "T3BlbkFJ" + "y" * 100, id="long-key-shaped-filler"), + ], +) +def test_openai_key_pattern_masks_a_key_glued_to_preceding_key_shaped_text(leading: str) -> None: + key: Final = "sk-proj-" + "c" * 74 + "T3BlbkFJ" + "d" * 74 + assert get_compiled_pattern("openai_api_key").sub("[KEY]", leading + key) == "[KEY][KEY]" + + +@pytest.mark.parametrize("prefix", ["sk-proj-", "sk-svcacct-", "sk-admin-", "sk-None-"]) +def test_openai_key_pattern_scans_repeated_prefixes_without_stalling(prefix: str) -> None: + prompt: Final = prefix * (128_000 // len(prefix)) + started: Final = time.perf_counter() + match: Final = get_compiled_pattern("openai_api_key").search(prompt) + elapsed: Final = time.perf_counter() - started + assert match is None + assert elapsed < 1.0 From 02d610f1a9e410e2f43d24431083f84f7d2158bd Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 7 Nov 2026 12:51:37 -0800 Subject: [PATCH 12/17] fix(anthropic): disable optional default thinking for reasoning effort none (#50541) --- .../crates/inference-messages/src/types.rs | 1 + .../crates/llms/src/anthropic/common_utils.rs | 1 + .../llms/src/anthropic/messages/thinking.rs | 37 +++++++++++++++++-- .../llms/src/base_llm/messages/context.rs | 3 ++ litellm/llms/anthropic/chat/transformation.py | 16 ++++++-- .../messages/native/transformation.py | 12 ++++-- .../bedrock/chat/converse_transformation.py | 8 ++-- .../llms/databricks/chat/transformation.py | 8 ++-- .../rust_bridge/host/model_capabilities.py | 1 + .../test_anthropic_messages_transformation.py | 14 +++++++ .../test_anthropic_reasoning_effort.py | 31 ++++++++++++++++ .../chat/test_converse_transformation.py | 15 ++++++++ .../test_databricks_chat_transformation.py | 15 ++++++++ .../host/test_model_capabilities.py | 13 +++++++ 14 files changed, 158 insertions(+), 17 deletions(-) diff --git a/litellm-rust/crates/inference-messages/src/types.rs b/litellm-rust/crates/inference-messages/src/types.rs index ae346259559..ece17b3380c 100644 --- a/litellm-rust/crates/inference-messages/src/types.rs +++ b/litellm-rust/crates/inference-messages/src/types.rs @@ -144,6 +144,7 @@ mod tests { supports_reasoning: true, supports_adaptive_thinking: true, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: true, supports_sampling_params: false, diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index 13653f438f7..0854820358b 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -1913,6 +1913,7 @@ mod tests { supports_reasoning: false, supports_adaptive_thinking: false, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: false, supports_sampling_params: true, diff --git a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs index ae3488a21f9..0ddbc915154 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs @@ -2,8 +2,8 @@ use litellm_llms_types::{ formats::{ chat_completions::ReasoningEffort, messages::{ - AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, - ThinkingConfig, ThinkingDisplay, + AdaptiveThinking, DisabledThinking, EffortLevel, MessagesOptionalParams, + MessagesRequest, OutputConfig, ThinkingConfig, ThinkingDisplay, }, }, recognized::Recognized, @@ -172,7 +172,11 @@ fn translate_reasoning_effort( ) else { return Ok(MessagesRequest { params: MessagesOptionalParams { - thinking: None, + thinking: (context.capabilities.thinking_on_by_default + && !context.capabilities.thinking_always_on) + .then(|| { + Recognized::Known(ThinkingConfig::Disabled(DisabledThinking::default())) + }), output_config: None, ..request.params }, @@ -748,6 +752,33 @@ mod tests { ); } + #[rstest] + #[case::default_thinking(true, false, Some(serde_json::json!({"type": "disabled"}))) ] + #[case::legacy(false, false, None)] + #[case::always_on(true, true, None)] + fn none_disables_only_optional_default_thinking( + #[case] thinking_on_by_default: bool, + #[case] thinking_always_on: bool, + #[case] thinking: Option, + ) { + let capabilities = MessagesModelCapabilities { + thinking_on_by_default, + thinking_always_on, + ..opus_4_7() + }; + let expected = match thinking { + Some(thinking) => serde_json::json!({"max_tokens": 1024, "thinking": thinking}), + None => serde_json::json!({"max_tokens": 1024}), + }; + assert_eq!( + translate( + capabilities, + serde_json::json!({"max_tokens": 1024, "reasoning_effort": "none"}) + ), + Ok(request(expected)) + ); + } + #[rstest] #[case::adaptive_model(opus_4_7())] #[case::effort_capable_model(opus_4_5())] diff --git a/litellm-rust/crates/llms/src/base_llm/messages/context.rs b/litellm-rust/crates/llms/src/base_llm/messages/context.rs index b514e975aa4..3d8b0f41d8c 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/context.rs @@ -32,6 +32,8 @@ pub struct MessagesModelCapabilities { #[serde(default)] pub thinking_always_on: bool, #[serde(default)] + pub thinking_on_by_default: bool, + #[serde(default)] pub supports_legacy_thinking: bool, #[serde(default)] pub supports_output_config: bool, @@ -55,6 +57,7 @@ impl Default for MessagesModelCapabilities { supports_reasoning: false, supports_adaptive_thinking: false, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: false, supports_sampling_params: true, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index e0d313175b7..48bccc7c001 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1718,7 +1718,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): llm_provider: str = "anthropic", ) -> AnthropicThinkingParam | None: """Capability probes read the cost map under ``custom_llm_provider``; ``llm_provider`` only tags raised exceptions.""" - if reasoning_effort is None or reasoning_effort == "none": + if reasoning_effort is None: + return None + if reasoning_effort == "none": + if AnthropicModelInfo.thinks_by_default( + model, custom_llm_provider + ) and not AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider): + return AnthropicThinkingParam(type="disabled") return None if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider): # without display, Anthropic defaults adaptive thinking to @@ -2126,12 +2132,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): custom_llm_provider=self._resolved_provider, llm_provider=self._resolved_provider, ) - if mapped_thinking is None: + if mapped_thinking is None or mapped_thinking.get("type") == "disabled": optional_params.pop("thinking", None) optional_params.pop("output_config", None) - else: + if mapped_thinking is not None: optional_params["thinking"] = mapped_thinking - if AnthropicConfig._is_adaptive_thinking_model(model, self._resolved_provider): + if mapped_thinking.get("type") != "disabled" and AnthropicConfig._is_adaptive_thinking_model( + model, self._resolved_provider + ): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(effort_value) if mapped_effort is None: AnthropicConfig._raise_invalid_reasoning_effort( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 5281ad99620..aa0693d6f03 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -471,7 +471,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """Map OpenAI-style ``reasoning_effort`` to native Anthropic params. Caller-supplied ``thinking`` / ``output_config`` win over the alias. - ``effort='none'`` clears both. Invalid efforts raise a 400. A mapped + ``effort='none'`` disables optional default thinking and clears effort. Invalid efforts raise a 400. A mapped thinking budget is capped below ``max_tokens`` and dropped when even the minimum budget cannot fit. """ @@ -485,6 +485,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if not isinstance(reasoning_effort, str): return + if reasoning_effort == "none": + optional_params.pop("thinking", None) + optional_params.pop("output_config", None) + try: mapped_thinking: Final = AnthropicConfig.map_reasoning_effort( reasoning_effort=reasoning_effort, @@ -495,8 +499,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): raise AnthropicError(message=str(e.message), status_code=400) if mapped_thinking is None: - optional_params.pop("thinking", None) - optional_params.pop("output_config", None) return fitted_thinking: Final = AnthropicConfig.cap_thinking_budget_to_max_tokens(mapped_thinking, max_tokens) @@ -505,7 +507,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return optional_params.setdefault("thinking", fitted_thinking) - if AnthropicModelInfo.is_adaptive_thinking_model(model, custom_llm_provider): + if fitted_thinking.get("type") != "disabled" and AnthropicModelInfo.is_adaptive_thinking_model( + model, custom_llm_provider + ): mapped_effort: Final = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort) if mapped_effort is None: raise AnthropicError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 12c5031628b..5c82a8d6bc4 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -583,12 +583,14 @@ class AmazonConverseConfig(BaseConfig): custom_llm_provider="bedrock", llm_provider="bedrock_converse", ) - if mapped_thinking is None: + if mapped_thinking is None or mapped_thinking.get("type") == "disabled": optional_params.pop("thinking", None) optional_params.pop("output_config", None) - else: + if mapped_thinking is not None: optional_params["thinking"] = mapped_thinking - if AnthropicConfig.is_adaptive_thinking_model(model, "bedrock"): + if mapped_thinking.get("type") != "disabled" and AnthropicConfig.is_adaptive_thinking_model( + model, "bedrock" + ): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort) if mapped_effort is None: AnthropicConfig.raise_invalid_reasoning_effort( diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index a173cdb50f2..7917a74247e 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -629,12 +629,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): custom_llm_provider="databricks", llm_provider="databricks", ) - if mapped_thinking is None: + if mapped_thinking is None or mapped_thinking.get("type") == "disabled": optional_params.pop("thinking", None) optional_params.pop("output_config", None) - else: + if mapped_thinking is not None: optional_params["thinking"] = mapped_thinking - if AnthropicConfig._is_adaptive_thinking_model(model, "databricks"): + if mapped_thinking.get("type") != "disabled" and AnthropicConfig._is_adaptive_thinking_model( + model, "databricks" + ): mapped_effort: str | None = None if isinstance(reasoning_effort_value, str): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort_value) diff --git a/litellm/rust_bridge/host/model_capabilities.py b/litellm/rust_bridge/host/model_capabilities.py index cbf59dfb97b..39eb0b08aa2 100644 --- a/litellm/rust_bridge/host/model_capabilities.py +++ b/litellm/rust_bridge/host/model_capabilities.py @@ -27,6 +27,7 @@ def anthropic_model_capabilities(model: str, custom_llm_provider: str | None) -> "supports_reasoning": supports("supports_reasoning"), "supports_adaptive_thinking": supports("supports_adaptive_thinking"), "thinking_always_on": supports("thinking_always_on"), + "thinking_on_by_default": AnthropicModelInfo.thinks_by_default(model, provider), "supports_legacy_thinking": supports("supports_legacy_thinking"), "supports_output_config": supports("supports_output_config"), "supports_sampling_params": AnthropicModelInfo._supports_sampling_params(resolved_model), # pyright: ignore[reportPrivateUsage] # same gate the handler applies diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 54b29ca6aa7..8f86888728f 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -721,3 +721,17 @@ def test_a_first_party_messages_request_never_fetches_the_remote_beta_headers_co assert request["messages"][1] == {"role": "system", "content": "be terse"} assert not remote_config.called + + +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-sonnet-5", {"type": "disabled"}), + ("claude-haiku-4-5", None), + ("claude-sonnet-5-5", None), + ], +) +def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None: + params: Final[dict[str, object]] = {"reasoning_effort": "none", "output_config": {"effort": "high"}} + AnthropicMessagesConfig._translate_reasoning_effort_to_anthropic(model, params, 1024, "anthropic") + assert params == ({"thinking": expected} if expected is not None else {}) diff --git a/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py b/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py index 573f586de1e..1b066e49e3b 100644 --- a/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py +++ b/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py @@ -5,11 +5,42 @@ Verifies that reasoning_effort=None returns None for all models, including Claude Opus 4.6. """ +from typing import Final + import pytest +from litellm.exceptions import BadRequestError from litellm.llms.anthropic.chat.transformation import AnthropicConfig +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-sonnet-5", {"type": "disabled"}), + ("claude-opus-5", {"type": "disabled"}), + ("claude-haiku-5-5", {"type": "disabled"}), + ("claude-haiku-4-5", None), + ("claude-sonnet-5-5", None), + ("claude-opus-5-5", None), + ("claude-fable-5", None), + ("claude-mythos-5", None), + ], +) +def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None: + mapped: Final = AnthropicConfig().map_openai_params({"reasoning_effort": "none"}, {}, model, False) + assert mapped.get("thinking") == expected + assert "output_config" not in mapped + + +@pytest.mark.parametrize("effort", [None, "bogus"]) +def test_default_thinking_preserves_absent_and_invalid_effort(effort: str | None) -> None: + if effort is None: + assert AnthropicConfig.map_reasoning_effort(effort, "claude-sonnet-5", "anthropic") is None + else: + with pytest.raises(BadRequestError, match="Invalid reasoning_effort"): + AnthropicConfig().map_openai_params({"reasoning_effort": effort}, {}, "claude-sonnet-5", False) + + class TestMapReasoningEffort: def test_none_returns_none_for_opus_4_6(self): """reasoning_effort=None should return None for Opus 4.6, not adaptive.""" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 4dcf312737d..23ac5baf1ba 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -12203,6 +12203,21 @@ def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hin assert messages == messages_before +@pytest.mark.parametrize( + "model, expected", + [ + ("us.anthropic.claude-sonnet-5", {"type": "disabled"}), + ("us.anthropic.claude-haiku-4-5", None), + ("us.anthropic.claude-sonnet-5-5", None), + ], +) +def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None: + params: Final[dict[str, object]] = {} + AmazonConverseConfig()._handle_reasoning_effort_parameter(model, "none", params) + assert params.get("thinking") == expected + assert "output_config" not in params + + def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None ) -> None: diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index e04dd35f851..053d60ef0e1 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -3220,3 +3220,18 @@ def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_co assert "cache_control" not in json.dumps(body["messages"][0]) assert json.dumps(body["messages"]).count('"cache_control"') == 3 + + +@pytest.mark.parametrize( + "model, expected", + [ + ("databricks-claude-sonnet-5", {"type": "disabled"}), + ("databricks-claude-haiku-4-5", None), + ("databricks-claude-sonnet-5-5", None), + ], +) +def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None: + params: Final = _map_reasoning_effort(model, "none") + assert params.get("thinking") == expected + assert "output_config" not in params + assert "reasoning_effort" not in params diff --git a/tests/unit/rust_bridge/host/test_model_capabilities.py b/tests/unit/rust_bridge/host/test_model_capabilities.py index 4d73a29d078..dffa1b2cde3 100644 --- a/tests/unit/rust_bridge/host/test_model_capabilities.py +++ b/tests/unit/rust_bridge/host/test_model_capabilities.py @@ -73,3 +73,16 @@ def test_capability_source_observes_runtime_registration_changes(monkeypatch: py assert before["supports_output_config"] is True assert after["supports_reasoning"] is False assert after["supports_output_config"] is False + + +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-sonnet-5", True), + ("claude-haiku-4-5", False), + ("claude-sonnet-5-5", True), + ], +) +def test_default_thinking_capability_is_projected(model: str, expected: bool) -> None: + capabilities: Final = anthropic_model_capabilities(model, "anthropic") + assert capabilities["thinking_on_by_default"] is expected From 093caa52fa063ad7cbb25c827531d02bde485f57 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 7 Nov 2026 13:16:39 -0800 Subject: [PATCH 13/17] test(e2e): add a Codex row where a sub-agent must read the task its parent hands it (#50494) --- tests/e2e/AGENTS.md | 2 +- tests/e2e/codex/harness.py | 39 ++++++++- tests/e2e/codex/rollout.py | 62 +++++++++++-- tests/e2e/codex/test_subagent_e2e.py | 31 +++++++ tests/e2e_harness/codex/test_codex_rollout.py | 87 +++++++++++++++++-- 5 files changed, 205 insertions(+), 16 deletions(-) create mode 100644 tests/e2e/codex/test_subagent_e2e.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index a4c51a53098..89bf24016da 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite -- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` +- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn, and a sub-agent, where Codex's multi-agent v2 hands a helper a random codeword as its task with shell tools off so the helper can only learn it from the task, and the helper's own final reply is read from its rollout), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `subagents` reads the sub-agents a thread spawned from their own rollouts, whose `session_meta` names that thread as their parent, each with its final replies and whether it forked the parent's conversation (`forked_from_id`), so the sub-agent row can reject a helper that saw the codeword in inherited history instead of its task. `exec` takes `enable` and `disable`, Codex feature names it passes as `--enable` and `--disable`. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only diff --git a/tests/e2e/codex/harness.py b/tests/e2e/codex/harness.py index df005b5b5e7..cc29581f2ce 100644 --- a/tests/e2e/codex/harness.py +++ b/tests/e2e/codex/harness.py @@ -12,7 +12,7 @@ from typing import Final from cli_harness.lockdown import isolated_home, locked_down_env from cli_harness.rate_limiter import RateBucket, RateLimiter -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import Compaction, Subagent, rollout_compactions, rollout_parent_thread, rollout_subagent from codex.run import CodexRun, codex_run from e2e_metadata import step @@ -71,8 +71,28 @@ class CodexHarness: _ = config.write(_mcp_server_table(name, server)) @step('Run Codex on {model} through the proxy with the prompt "{prompt}"') - def exec(self, prompt: str, *, model: str, cwd: Path, compact_after_tokens: int | None = None) -> CodexRun: - return self._run(("exec",), (*_compaction_flags(compact_after_tokens), "--model", model, "--", prompt), cwd) + def exec( + self, + prompt: str, + *, + model: str, + cwd: Path, + compact_after_tokens: int | None = None, + enable: tuple[str, ...] = (), + disable: tuple[str, ...] = (), + ) -> CodexRun: + return self._run( + ("exec",), + ( + *_compaction_flags(compact_after_tokens), + *_feature_flags(enable=enable, disable=disable), + "--model", + model, + "--", + prompt, + ), + cwd, + ) @step('Resume Codex thread {thread_id} on {model} through the proxy with the prompt "{prompt}"') def resume( @@ -88,6 +108,12 @@ class CodexHarness: rollouts: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl")) return tuple(chain.from_iterable(rollout_compactions(rollout.read_text()) for rollout in rollouts)) + def subagents(self, thread_id: str) -> tuple[Subagent, ...]: + rollouts: Final = tuple( + path.read_text() for path in sorted(self.codex_home.glob("sessions/**/rollout-*.jsonl")) + ) + return tuple(rollout_subagent(rollout) for rollout in rollouts if rollout_parent_thread(rollout) == thread_id) + def _run(self, subcommand: tuple[str, ...], args: tuple[str, ...], cwd: Path) -> CodexRun: _ = self.limiter.acquire(self.rate_bucket) completed: Final = subprocess.run( @@ -116,6 +142,13 @@ def _mcp_server_table(name: str, server: McpServer) -> str: ) +def _feature_flags(*, enable: tuple[str, ...], disable: tuple[str, ...]) -> tuple[str, ...]: + return ( + *chain.from_iterable(("--enable", feature) for feature in enable), + *chain.from_iterable(("--disable", feature) for feature in disable), + ) + + def _compaction_flags(after_tokens: int | None) -> tuple[str, ...]: match after_tokens: case None: diff --git a/tests/e2e/codex/rollout.py b/tests/e2e/codex/rollout.py index fcaeaa7ca42..5b8253fab10 100644 --- a/tests/e2e/codex/rollout.py +++ b/tests/e2e/codex/rollout.py @@ -1,19 +1,23 @@ -"""The compactions Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. +"""What Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. -`codex exec --json` prints no event for a compaction, so the rollout is the only place it shows. Each line is -`{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only `compacted` lines are read. +`codex exec --json` prints no event for a compaction or for a sub-agent's own turn, so the rollout is the only place +they show. Each line is `{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`). A remote compaction is one whose replacement history carries the provider's `compaction` item, while a local one -replaces the history with a summary Codex wrote itself. +replaces the history with a summary Codex wrote itself. A sub-agent gets a rollout of its own whose `session_meta` +names the thread that spawned it, and carries `forked_from_id` when it inherited that thread's conversation. Each turn +it finishes ends in a `task_complete` event with its last message. """ from __future__ import annotations from dataclasses import dataclass -from typing import Final +from typing import Final, Literal from pydantic import BaseModel, ConfigDict _COMPACTED: Final = "compacted" +_SESSION_META: Final = "session_meta" +_EVENT: Final = "event_msg" _PROVIDER_COMPACTION_ITEM: Final = "compaction" @@ -34,14 +38,58 @@ class _CompactedPayload(_Frozen): replacement_history: tuple[_HistoryItem, ...] | None = None +class _SessionMeta(_Frozen): + parent_thread_id: str | None = None + forked_from_id: str | None = None + + +class _Event(_Frozen): + type: str + + +class _TaskComplete(_Frozen): + type: Literal["task_complete"] + last_agent_message: str | None = None + + @dataclass(frozen=True, slots=True) class Compaction: remote: bool +@dataclass(frozen=True, slots=True) +class Subagent: + forked: bool + replies: tuple[str, ...] + + def rollout_compactions(rollout: str) -> tuple[Compaction, ...]: - lines: Final = tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) - return tuple(_compaction(line.payload) for line in lines if line.type == _COMPACTED) + return tuple(_compaction(line.payload) for line in _lines(rollout) if line.type == _COMPACTED) + + +def rollout_parent_thread(rollout: str) -> str | None: + return _session_meta(rollout).parent_thread_id + + +def rollout_subagent(rollout: str) -> Subagent: + return Subagent(forked=_session_meta(rollout).forked_from_id is not None, replies=rollout_final_messages(rollout)) + + +def rollout_final_messages(rollout: str) -> tuple[str, ...]: + events: Final = (line.payload for line in _lines(rollout) if line.type == _EVENT) + completions: Final = ( + _TaskComplete.model_validate(event) for event in events if _Event.model_validate(event).type == "task_complete" + ) + return tuple(done.last_agent_message for done in completions if done.last_agent_message is not None) + + +def _session_meta(rollout: str) -> _SessionMeta: + meta: Final = next((line.payload for line in _lines(rollout) if line.type == _SESSION_META), None) + return _SessionMeta.model_validate(meta) if meta is not None else _SessionMeta() + + +def _lines(rollout: str) -> tuple[_Line, ...]: + return tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) def _compaction(payload: object) -> Compaction: diff --git a/tests/e2e/codex/test_subagent_e2e.py b/tests/e2e/codex/test_subagent_e2e.py new file mode 100644 index 00000000000..0ad86c8b49a --- /dev/null +++ b/tests/e2e/codex/test_subagent_e2e.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from codex.harness import CodexHarness +from codex.setups import SETUPS, ModelSetup +from e2e_config import unique_marker + +pytestmark = pytest.mark.e2e + + +@pytest.mark.parametrize("deployment", SETUPS) +class TestCodexSubagent: + def test_a_subagent_reads_the_task_its_parent_hands_it(self, codex: CodexHarness, setup: ModelSetup) -> None: + codeword: Final = unique_marker() + run: Final = codex.exec( + 'Spawn exactly one sub-agent with the spawn_agent tool, with fork_turns set to "none" so it does not' + f" inherit this conversation. Its task: reply with exactly this codeword and nothing else: {codeword}." + " Wait for its reply, then tell me the exact reply it sent you, verbatim.", + model=setup.model, + cwd=codex.new_workspace(files={}), + enable=("multi_agent_v2",), + disable=("shell_tool", "unified_exec"), + ) + assert run.exit_code == 0, run + assert run.thread_id is not None, run + subagents: Final = codex.subagents(run.thread_id) + assert subagents, run + assert not any(subagent.forked for subagent in subagents), (subagents, run) + assert any(any(codeword in reply for reply in subagent.replies) for subagent in subagents), (subagents, run) diff --git a/tests/e2e_harness/codex/test_codex_rollout.py b/tests/e2e_harness/codex/test_codex_rollout.py index 2ef27a8a9c5..3db39066204 100644 --- a/tests/e2e_harness/codex/test_codex_rollout.py +++ b/tests/e2e_harness/codex/test_codex_rollout.py @@ -8,7 +8,14 @@ from typing import Final from cli_harness.rate_limiter import RateBucket, RateLimiter from codex.harness import CodexHarness -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import ( + Compaction, + Subagent, + rollout_compactions, + rollout_final_messages, + rollout_parent_thread, + rollout_subagent, +) THREAD_ID: Final = "01a18fea-b2e8-7ed2-a29b-1ce4d2b138c3" USER_PROMPT: Final = { @@ -60,13 +67,83 @@ class TestRolloutCompactions: assert rollout_compactions(_rollout()) == () -def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: - codex: Final = CodexHarness( - home=tmp_path, +def _harness(home: Path) -> CodexHarness: + return CodexHarness( + home=home, api_key="unused", rate_bucket=RateBucket.OPENAI, - limiter=RateLimiter({}, tmp_path, time.monotonic, time.sleep), + limiter=RateLimiter({}, home, time.monotonic, time.sleep), ) + + +def _subagent_rollout( + *, thread_id: str, parent: str | None, final_messages: tuple[str | None, ...], forked: bool = False +) -> str: + meta: Final = ( + {"id": thread_id, "cli_version": "0.165.0"} + | ({"parent_thread_id": parent} if parent else {}) + | ({"forked_from_id": parent} if forked else {}) + ) + inherited: Final = (_line("session_meta", {"id": parent, "cli_version": "0.165.0"}),) if forked else () + return "\n".join( + ( + _line("session_meta", meta), + *inherited, + _line("event_msg", {"type": "task_started", "turn_id": "t1"}), + _line( + "response_item", + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "thinking"}]}, + ), + *( + _line("event_msg", {"type": "task_complete", "turn_id": "t1", "last_agent_message": message}) + for message in final_messages + ), + ) + ) + + +class TestRolloutSubagents: + def test_a_subagent_rollout_names_the_thread_that_spawned_it(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=()) + assert rollout_parent_thread(rollout) == THREAD_ID + + def test_a_top_level_rollout_has_no_parent(self) -> None: + assert rollout_parent_thread(_rollout()) is None + + def test_a_subagent_that_inherited_its_parents_conversation_is_forked(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("hi",), forked=True) + assert rollout_subagent(rollout) == Subagent(forked=True, replies=("hi",)) + + def test_a_subagent_that_started_fresh_is_not_forked(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("hi",)) + assert rollout_subagent(rollout) == Subagent(forked=False, replies=("hi",)) + + def test_the_final_messages_are_the_last_message_of_each_finished_turn(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("first", None, "last")) + assert rollout_final_messages(rollout) == ("first", "last") + + +def test_the_harness_reads_only_the_subagents_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: + codex: Final = _harness(tmp_path) + sessions: Final = codex.codex_home / "sessions" / "2026" / "11" / "06" + sessions.mkdir(parents=True) + rollouts: Final = { + THREAD_ID: _subagent_rollout(thread_id=THREAD_ID, parent=None, final_messages=("the parent's answer",)), + "child-a": _subagent_rollout(thread_id="child-a", parent=THREAD_ID, final_messages=("reply a",)), + "child-b": _subagent_rollout(thread_id="child-b", parent=THREAD_ID, final_messages=("reply b",), forked=True), + "stranger": _subagent_rollout(thread_id="stranger", parent="other-thread", final_messages=("not ours",)), + } + for thread_id, rollout in rollouts.items(): + _ = (sessions / f"rollout-2026-11-06T17-01-05-{thread_id}.jsonl").write_text(rollout) + + assert codex.subagents(THREAD_ID) == ( + Subagent(forked=False, replies=("reply a",)), + Subagent(forked=True, replies=("reply b",)), + ) + + +def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: + codex: Final = _harness(tmp_path) sessions: Final = codex.codex_home / "sessions" / "2026" / "10" / "30" sessions.mkdir(parents=True) _ = (sessions / f"rollout-2026-10-30T20-03-32-{THREAD_ID}.jsonl").write_text(_rollout(REMOTE)) From 152a5cac7b26414f948ef553283403502e5189fc Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Sat, 7 Nov 2026 13:25:59 -0800 Subject: [PATCH 14/17] feat(ui): set deployment weights in key and team router settings (#50552) * feat(ui): set deployment weights in key and team router settings The proxy already applies router_settings.weights per key and team, but no dashboard screen could set them, and saving a team in the UI dropped weights set through the API Co-Authored-By: Claude Opus 5.5 * fix(ui): only offer deployments the key or team may weight Co-Authored-By: Claude Opus 5.5 * fix(ui): show weight load errors, inherited strategy and team deployments on key create Co-Authored-By: Claude Opus 5.5 * fix(ui): keep reliability fields on create forms and label weight inputs by deployment id Co-Authored-By: Claude Opus 5.5 * style(ui): name the accordion value identity object Co-Authored-By: Claude Opus 5.5 * test(ui): expect weights in the key create router settings payload Co-Authored-By: Claude Opus 5.5 * fix(ui): keep weight drafts, loading state and team switches correct Co-Authored-By: Claude Opus 5.5 * fix(ui): only accept weight keystrokes that save as a number Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- ...uterSettingsAccordion.integration.test.tsx | 44 ++++ .../RouterSettingsAccordion.test.tsx | 189 +++++++++++++++- .../RouterSettingsAccordion.tsx | 58 ++++- .../RouterSettingsSummary.test.tsx | 9 + .../RouterSettingsSummary.tsx | 24 ++ .../RouterWeightsSection.tsx | 206 ++++++++++++++++++ .../routerSettingsPayload.test.ts | 9 + .../routerSettingsPayload.ts | 1 + .../common_components/routerWeights.test.ts | 152 +++++++++++++ .../common_components/routerWeights.ts | 105 +++++++++ .../create_key_button.integration.test.tsx | 1 + .../organisms/create_key_button.tsx | 1 + .../src/components/team/TeamInfo.tsx | 2 + ui/litellm-dashboard/src/i18n/locales/en.json | 16 ++ 14 files changed, 804 insertions(+), 13 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx create mode 100644 ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts create mode 100644 ui/litellm-dashboard/src/components/common_components/routerWeights.ts diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx new file mode 100644 index 00000000000..0f23e91207a --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx @@ -0,0 +1,44 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, render, screen } from "@testing-library/react"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; +import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion"; + +vi.mock("../networking", () => ({ + getRouterSettingsCall: vi.fn().mockResolvedValue({}), + modelInfoCall: vi.fn().mockResolvedValue({ data: [] }), +})); + +vi.mock("@/components/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]), + fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]), +})); + +const EchoingCreateForm = () => { + const [value, setValue] = useState(undefined); + return ; +}; + +describe("RouterSettingsAccordion on a create form", () => { + it("should render every reliability input after the parent echoes the first value back", async () => { + render( + + + , + ); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 250)); + }); + + for (const field of [ + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "retry_after", + "model_group_alias", + ]) { + expect(screen.getByRole("textbox", { name: new RegExp(field) })).toBeEnabled(); + } + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx index 6f819607e3f..827c1b60636 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx @@ -1,13 +1,18 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; -import type { ReactElement, ReactNode } from "react"; +import { createRef, useState, type ReactElement, type ReactNode } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm"; import { fetchAvailableModels, fetchAvailableModelsForTeam } from "@/components/llm_calls/fetch_models"; -import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion"; +import { getRouterSettingsCall, modelInfoCall } from "../networking"; +import RouterSettingsAccordion, { + RouterSettingsAccordionRef, + RouterSettingsAccordionValue, +} from "./RouterSettingsAccordion"; vi.mock("../networking", () => ({ getRouterSettingsCall: vi.fn().mockResolvedValue({}), + modelInfoCall: vi.fn(), })); vi.mock("@/components/llm_calls/fetch_models", () => ({ @@ -130,6 +135,186 @@ describe("RouterSettingsAccordion", () => { expect(screen.getByTestId("available-models")).not.toHaveTextContent("slow-team-model"); }); + describe("deployment weights", () => { + const gpt5Deployments = { + data: [ + { model_name: "gpt-5", litellm_params: { model: "openai/gpt-5" }, model_info: { id: "dep-openai" } }, + { model_name: "gpt-5", litellm_params: { model: "azure/gpt-5" }, model_info: { id: "dep-azure" } }, + ], + }; + + beforeEach(() => { + vi.mocked(modelInfoCall).mockResolvedValue(gpt5Deployments); + }); + + it("should save stored weights together with a newly entered one", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + + fireEvent.change(await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)"), { target: { value: "1" } }); + + expect(await screen.findByText("90% of traffic")).toBeInTheDocument(); + expect(screen.getByText("10% of traffic")).toBeInTheDocument(); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ + "gpt-5": { "dep-openai": 9, "dep-azure": 1 }, + }); + expect(vi.mocked(modelInfoCall).mock.calls[0][7]).toBe("team-123"); + expect(vi.mocked(modelInfoCall).mock.calls[0][11]).toBe("gpt-5"); + }); + + it("should save null once every weight is cleared, so the stored weights are removed", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + + fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } }); + + await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull()); + }); + + it("should keep a model group on screen when the parent echoes back the value it was sent", async () => { + const EchoingParent = () => { + const [value, setValue] = useState({ + router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } }, + }); + return ; + }; + renderWithQueryClient(); + + fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } }); + await act(async () => { + vi.advanceTimersByTime(200); + }); + + expect(screen.getByLabelText("Weight for openai/gpt-5 (dep-openai)")).toHaveValue(""); + expect(screen.getByRole("button", { name: "Remove weights for gpt-5" })).toBeInTheDocument(); + }); + + it("should say the deployments failed to load instead of calling saved weights stale", async () => { + vi.mocked(modelInfoCall).mockRejectedValue(new Error("403")); + const ref = createRef(); + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Could not load this model group's deployments. Saved weights are kept."), + ).toBeInTheDocument(); + expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument(); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } }); + }); + + it("should warn that weights are ignored when the inherited global strategy is not simple-shuffle", async () => { + vi.mocked(getRouterSettingsCall).mockResolvedValueOnce({ + fields: [{ field_name: "routing_strategy", options: ["simple-shuffle"], field_value: "latency-based-routing" }], + }); + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Weights are ignored with latency-based-routing. Select simple-shuffle to use them."), + ).toBeInTheDocument(); + }); + + it("should accept a decimal weight typed one character at a time", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + const input = await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)"); + + fireEvent.change(input, { target: { value: "1." } }); + expect(input).toHaveValue("1."); + fireEvent.change(input, { target: { value: "1.5" } }); + + expect(input).toHaveValue("1.5"); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ + "gpt-5": { "dep-openai": 9, "dep-azure": 1.5 }, + }); + }); + + it("should ignore keystrokes that would not save as a weight, so the field always shows the saved value", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + const input = await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"); + + fireEvent.change(input, { target: { value: "9x" } }); + fireEvent.change(input, { target: { value: "-1" } }); + + expect(input).toHaveValue("9"); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } }); + }); + + it("should not call saved weights stale while the deployments are still loading", async () => { + vi.mocked(modelInfoCall).mockReturnValue(new Promise(() => {})); + renderWithQueryClient( + , + ); + + expect(await screen.findByText("dep-openai")).toBeInTheDocument(); + expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument(); + }); + + it("should drop weights picked for one team when the form switches to another team", async () => { + const ref = createRef(); + const value = { router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }; + const { rerender } = renderWithQueryClient( + , + ); + await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"); + + rerender(); + + await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull()); + }); + + it("should flag a group whose weights are all zero", async () => { + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Set at least one weight above 0, or remove this model group."), + ).toBeInTheDocument(); + }); + }); + it("does not call onChange when unmounted mid-wait", async () => { const onChange = vi.fn<(value: RouterSettingsAccordionValue) => void>(); const { unmount } = renderWithQueryClient(); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx index 68fbd294b7c..a89c4138970 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx @@ -9,6 +9,8 @@ import { Fallbacks } from "../Settings/RouterSettings/Fallbacks/AddFallbacks"; import { FallbackSelectionForm } from "../Settings/RouterSettings/Fallbacks/FallbackSelectionForm"; import { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig"; import { fetchAvailableModels, fetchAvailableModelsForTeam, ModelGroup } from "@/components/llm_calls/fetch_models"; +import RouterWeightsSection from "./RouterWeightsSection"; +import { RouterWeights, routerWeightsFrom, savedRouterWeights } from "./routerWeights"; export interface RouterSettingsAccordionValue { router_settings: { @@ -24,6 +26,7 @@ export interface RouterSettingsAccordionValue { model_group_alias?: { [key: string]: any } | null; enable_tag_filtering?: boolean; routing_strategy_args?: { [key: string]: any } | null; + weights?: RouterWeights | null; }; } @@ -41,20 +44,38 @@ export interface RouterSettingsAccordionRef { const PROPAGATE_WAIT_MS = 100; +const EMPTY_RELIABILITY_SETTINGS = { + allowed_fails: null, + cooldown_time: null, + num_retries: null, + timeout: null, + retry_after: null, + model_group_alias: null, +}; + +const valueKeyOf = (settings: RouterSettingsAccordionValue["router_settings"]): string => { + const { routing_strategy, fallbacks, enable_tag_filtering, weights } = settings; + const identity = { routing_strategy, fallbacks, enable_tag_filtering, weights }; + return JSON.stringify(identity); +}; + const RouterSettingsAccordion = forwardRef( ({ accessToken, value, onChange, modelData, teamId }, ref) => { const { t } = useTranslation(); const [formValue, setFormValue] = useState({ - routerSettings: {}, + routerSettings: EMPTY_RELIABILITY_SETTINGS, selectedStrategy: null, enableTagFiltering: false, }); const [fallbacks, setFallbacks] = useState([]); const [fallbackGroups, setFallbackGroups] = useState([]); + const [weights, setWeights] = useState({}); const [availableRoutingStrategies, setAvailableRoutingStrategies] = useState([]); + const [globalRoutingStrategy, setGlobalRoutingStrategy] = useState(null); const [routerFieldsMetadata, setRouterFieldsMetadata] = useState<{ [key: string]: any }>({}); const [routingStrategyDescriptions, setRoutingStrategyDescriptions] = useState<{ [key: string]: string }>({}); const isInternalUpdateRef = useRef(false); + const weightsTeamIdRef = useRef(teamId); const lastInitializedValueRef = useRef(null); // Convert fallbacks format to groups format @@ -90,13 +111,7 @@ const RouterSettingsAccordion = forwardRef { // Create a stable key from the value to detect actual external changes - const valueKey = value?.router_settings - ? JSON.stringify({ - routing_strategy: value.router_settings.routing_strategy, - fallbacks: value.router_settings.fallbacks, - enable_tag_filtering: value.router_settings.enable_tag_filtering, - }) - : null; + const valueKey = value?.router_settings ? valueKeyOf(value.router_settings) : null; // Skip if this is an internal update (from our own onChange) and the value hasn't actually changed if (isInternalUpdateRef.current && valueKey === lastInitializedValueRef.current) { @@ -118,7 +133,7 @@ const RouterSettingsAccordion = forwardRef { + if (weightsTeamIdRef.current === teamId) return; + weightsTeamIdRef.current = teamId; + setWeights({}); + }, [teamId]); + // Fetch router settings metadata useEffect(() => { if (!accessToken) { @@ -169,6 +192,7 @@ const RouterSettingsAccordion = forwardRef { setFallbackGroups(newGroups); @@ -360,6 +386,16 @@ const RouterSettingsAccordion = forwardRef +
+ +
{ expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument(); }); + it("should list each weighted model group with its deployment weights", () => { + render(); + + expect(screen.getByText("Deployment Weights:")).toBeInTheDocument(); + expect(screen.getByText("gpt-5")).toBeInTheDocument(); + expect(screen.getByText("dep-a: 9, dep-b: 1")).toBeInTheDocument(); + expect(screen.queryByText("empty")).not.toBeInTheDocument(); + }); + it("should show the empty state when every setting is null", () => { render(); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx index f99a154fe3d..89ec0873920 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx @@ -2,6 +2,7 @@ import { useTranslation } from "react-i18next"; import { Badge } from "@/components/ui/badge"; import i18n from "@/i18n/i18n"; import { hasRouterSettings } from "./routerSettingsPayload"; +import { routerWeightsFrom, savedRouterWeights } from "./routerWeights"; interface RouterSettingsSummaryProps { routerSettings: Record | null | undefined; @@ -15,6 +16,28 @@ const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => { ); }; +export function RouterWeightsSummary({ weights }: { weights: unknown }) { + const { t } = useTranslation(); + const saved = savedRouterWeights(routerWeightsFrom(weights)); + if (!saved) return null; + return ( +
+
{t("routerWeights.summary")}
+
+ {Object.entries(saved).map(([group, groupWeights]) => ( +
+ {group} + -> + {Object.entries(groupWeights) + .map(([deploymentId, weight]) => `${deploymentId}: ${weight}`) + .join(", ")} +
+ ))} +
+
+ ); +} + export default function RouterSettingsSummary({ routerSettings, emptyText = i18n.t("routerSettingsSummary.empty"), @@ -63,6 +86,7 @@ export default function RouterSettingsSummary({ )} + ); } diff --git a/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx b/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx new file mode 100644 index 00000000000..32c5e70a71f --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx @@ -0,0 +1,206 @@ +import { useQuery } from "@tanstack/react-query"; +import { AlertCircle, X } from "lucide-react"; +import { useId, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { modelInfoCall } from "../networking"; +import { + GroupWeights, + RouterWeights, + WeightableDeployment, + hasNoPositiveWeight, + parseWeightInput, + weightRows, + withDeploymentWeight, + withGroup, + withoutGroup, +} from "./routerWeights"; + +const WEIGHTED_STRATEGY = "simple-shuffle"; +const DEPLOYMENTS_PAGE_SIZE = 1000; + +interface RouterWeightsSectionProps { + accessToken: string; + teamId?: string | null; + availableModels: string[]; + selectedStrategy: string | null; + weights: RouterWeights; + onChange: (weights: RouterWeights) => void; +} + +interface WeightGroupProps { + accessToken: string; + teamId?: string | null; + group: string; + groupWeights: GroupWeights; + onWeightChange: (deploymentId: string, weight: number | null) => void; + onRemove: () => void; +} + +interface WeightInputProps { + label: string; + weight: number | null; + onChange: (weight: number | null) => void; +} + +const formatWeight = (weight: number | null): string => (weight === null ? "" : String(weight)); + +function WeightInput({ label, weight, onChange }: WeightInputProps) { + const [draft, setDraft] = useState(formatWeight(weight)); + const [shownWeight, setShownWeight] = useState(weight); + if (weight !== shownWeight) { + setShownWeight(weight); + if (parseWeightInput(draft) !== weight) setDraft(formatWeight(weight)); + } + + const handleChange = (event: React.ChangeEvent) => { + const parsed = parseWeightInput(event.target.value); + if (parsed === undefined) return; + setDraft(event.target.value); + onChange(parsed); + }; + + return ( + + ); +} + +function WeightGroup({ accessToken, teamId, group, groupWeights, onWeightChange, onRemove }: WeightGroupProps) { + const { t } = useTranslation(); + const { + data: deployments = [], + isLoading, + isError, + } = useQuery({ + queryKey: ["routerWeightDeployments", accessToken, teamId ?? null, group], + queryFn: async () => + ( + await modelInfoCall( + accessToken, + "", + "", + 1, + DEPLOYMENTS_PAGE_SIZE, + undefined, + undefined, + teamId ?? undefined, + undefined, + undefined, + undefined, + group, + ) + ).data ?? [], + enabled: Boolean(accessToken), + }); + const rows = weightRows(group, groupWeights, deployments, teamId ?? null); + const deploymentsKnown = !isLoading && !isError; + const noDeployments = deploymentsKnown && rows.length === 0; + + return ( +
+
+ {group} + +
+ {isError && ( +

+ {t("routerWeights.loadFailed")} +

+ )} + {noDeployments &&

{t("routerWeights.noDeployments")}

} + {rows.map((row) => ( +
+
+
+ {row.label ?? (deploymentsKnown ? t("routerWeights.unknownDeployment") : "")} +
+
{row.deploymentId}
+
+ onWeightChange(row.deploymentId, weight)} + /> + + {row.sharePercent === null ? "" : t("routerWeights.share", { percent: row.sharePercent })} + +
+ ))} + {hasNoPositiveWeight(groupWeights) && ( +

+ + {t("routerWeights.noPositiveWeight")} +

+ )} +
+ ); +} + +export default function RouterWeightsSection({ + accessToken, + teamId, + availableModels, + selectedStrategy, + weights, + onChange, +}: RouterWeightsSectionProps) { + const { t } = useTranslation(); + const addGroupInputId = useId(); + const unweightedModels = availableModels.filter((model) => !(model in weights)); + const ignoredByStrategy = Boolean(selectedStrategy) && selectedStrategy !== WEIGHTED_STRATEGY; + + return ( +
+
+

{t("routerWeights.title")}

+

{t("routerWeights.description")}

+
+ {ignoredByStrategy && Object.keys(weights).length > 0 && ( +
+ + {t("routerWeights.ignoredForStrategy", { strategy: selectedStrategy })} +
+ )} + {Object.entries(weights).map(([group, groupWeights]) => ( + + onChange(withDeploymentWeight(weights, group, deploymentId, weight)) + } + onRemove={() => onChange(withoutGroup(weights, group))} + /> + ))} +
+ + ({ label: model, value: model }))} + value={null} + onValueChange={(group) => { + if (group) onChange(withGroup(weights, group)); + }} + placeholder={t("routerWeights.addGroupPlaceholder")} + emptyText={t("routerWeights.noModels")} + /> +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts index 542ed17d7a2..05645a50b41 100644 --- a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts @@ -101,9 +101,18 @@ describe("routerSettingsUpdate", () => { model_group_alias: null, enable_tag_filtering: null, routing_strategy_args: null, + weights: null, }); }); + it("should replace stored weights with the edited ones, including clearing them", () => { + const stored = { weights: { "gpt-4": { "dep-a": 9, "dep-b": 1 } }, num_retries: 2 }; + expect(routerSettingsUpdate({ num_retries: 2, weights: { "gpt-4": { "dep-a": 1 } } }, stored)).toMatchObject({ + weights: { "gpt-4": { "dep-a": 1 } }, + }); + expect(routerSettingsUpdate({ num_retries: 2, weights: null }, stored)).toMatchObject({ weights: null }); + }); + it("should send the edited settings when the user configured something", () => { expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks }); }); diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts index 1d048425a6d..1bf6d0ef252 100644 --- a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts @@ -15,6 +15,7 @@ const EDITOR_OWNED_FIELDS: Record = { model_group_alias: true, enable_tag_filtering: true, routing_strategy_args: true, + weights: true, }; const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array; diff --git a/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts b/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts new file mode 100644 index 00000000000..3df42281bf5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; +import { + hasNoPositiveWeight, + parseWeightInput, + routerWeightsFrom, + savedRouterWeights, + weightRows, + withDeploymentWeight, + withGroup, + withoutGroup, +} from "./routerWeights"; + +const deployment = (id: string, modelName: string, model: string, team?: { id: string; publicName: string }) => ({ + model_name: modelName, + litellm_params: { model }, + model_info: { id, ...(team ? { team_id: team.id, team_public_model_name: team.publicName } : {}) }, +}); + +describe("routerWeightsFrom", () => { + it("should keep well-formed groups and drop malformed weights from the stored blob", () => { + expect( + routerWeightsFrom({ + "gpt-5": { a: 9, b: 1, bad: "1", negative: -1, inf: Infinity }, + broken: [1, 2], + }), + ).toStrictEqual({ "gpt-5": { a: 9, b: 1 } }); + }); + + it("should read missing or non-object weights as none", () => { + expect(routerWeightsFrom(undefined)).toStrictEqual({}); + expect(routerWeightsFrom(null)).toStrictEqual({}); + expect(routerWeightsFrom("x")).toStrictEqual({}); + }); +}); + +describe("weight edits", () => { + it("should set, replace and clear one deployment without touching others", () => { + const start = { "gpt-5": { a: 9 }, other: { c: 2 } }; + const added = withDeploymentWeight(start, "gpt-5", "b", 1); + expect(added).toStrictEqual({ "gpt-5": { a: 9, b: 1 }, other: { c: 2 } }); + expect(withDeploymentWeight(added, "gpt-5", "a", 3)).toStrictEqual({ "gpt-5": { a: 3, b: 1 }, other: { c: 2 } }); + expect(withDeploymentWeight(added, "gpt-5", "a", null)).toStrictEqual({ "gpt-5": { b: 1 }, other: { c: 2 } }); + expect(start).toStrictEqual({ "gpt-5": { a: 9 }, other: { c: 2 } }); + }); + + it("should add an empty group once and remove a group entirely", () => { + const withEmpty = withGroup({ "gpt-5": { a: 9 } }, "claude"); + expect(withEmpty).toStrictEqual({ "gpt-5": { a: 9 }, claude: {} }); + expect(withGroup(withEmpty, "gpt-5")).toBe(withEmpty); + expect(withoutGroup(withEmpty, "gpt-5")).toStrictEqual({ claude: {} }); + }); +}); + +describe("savedRouterWeights", () => { + it("should drop groups with no weights so they are not saved", () => { + expect(savedRouterWeights({ "gpt-5": { a: 0, b: 2 }, claude: {} })).toStrictEqual({ "gpt-5": { a: 0, b: 2 } }); + }); + + it("should save no weights as null so the field is cleared", () => { + expect(savedRouterWeights({})).toBeNull(); + expect(savedRouterWeights({ claude: {} })).toBeNull(); + }); +}); + +describe("hasNoPositiveWeight", () => { + it("should flag a group whose weights are all zero, as the proxy rejects it", () => { + expect(hasNoPositiveWeight({ a: 0, b: 0 })).toBe(true); + expect(hasNoPositiveWeight({ a: 0, b: 0.5 })).toBe(false); + expect(hasNoPositiveWeight({})).toBe(false); + }); +}); + +describe("weightRows", () => { + it("should list the group's deployments with each one's share of traffic", () => { + const rows = weightRows( + "gpt-5", + { a: 9, b: 1 }, + [ + deployment("a", "gpt-5", "openai/gpt-5"), + deployment("b", "gpt-5", "azure/gpt-5"), + deployment("c", "gpt-5", "bedrock/gpt-5"), + ], + null, + ); + expect(rows).toStrictEqual([ + { deploymentId: "a", label: "openai/gpt-5", weight: 9, sharePercent: 90 }, + { deploymentId: "b", label: "azure/gpt-5", weight: 1, sharePercent: 10 }, + { deploymentId: "c", label: "bedrock/gpt-5", weight: null, sharePercent: 0 }, + ]); + }); + + it("should match team deployments by their public model name, not their internal routing name", () => { + const rows = weightRows( + "gpt-5", + {}, + [ + deployment("team-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }), + deployment("other-group", "gpt-5-mini", "openai/gpt-5-mini"), + ], + "team-1", + ); + expect(rows.map((row) => row.deploymentId)).toStrictEqual(["team-dep"]); + }); + + it("should group a deployment with no team by its model name even if it carries a public name", () => { + const orphanedPublicName = { + model_name: "gpt-5-internal", + litellm_params: { model: "openai/gpt-5" }, + model_info: { id: "x", team_public_model_name: "gpt-5" }, + }; + expect(weightRows("gpt-5", {}, [orphanedPublicName], null)).toStrictEqual([]); + expect(weightRows("gpt-5-internal", {}, [orphanedPublicName], null)).toHaveLength(1); + }); + + it("should leave out another team's deployments, which the proxy rejects", () => { + const deployments = [ + deployment("global", "gpt-5", "openai/gpt-5"), + deployment("team-1-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }), + deployment("team-2-dep", "model_name_team-2_def", "openai/gpt-5", { id: "team-2", publicName: "gpt-5" }), + ]; + expect(weightRows("gpt-5", {}, deployments, "team-1").map((row) => row.deploymentId)).toStrictEqual([ + "global", + "team-1-dep", + ]); + expect(weightRows("gpt-5", {}, deployments, null).map((row) => row.deploymentId)).toStrictEqual(["global"]); + }); + + it("should show no share until some weight is set", () => { + const rows = weightRows("gpt-5", {}, [deployment("a", "gpt-5", "openai/gpt-5")], null); + expect(rows[0].sharePercent).toBeNull(); + }); + + it("should keep a weighted id whose deployment is gone so it can still be cleared", () => { + const rows = weightRows("gpt-5", { a: 1, deleted: 1 }, [deployment("a", "gpt-5", "openai/gpt-5")], null); + const staleRow = { deploymentId: "deleted", label: null, weight: 1, sharePercent: 50 }; + expect(rows[1]).toStrictEqual(staleRow); + }); +}); + +describe("parseWeightInput", () => { + it("should accept non-negative numbers, clear on empty and reject the rest", () => { + expect(parseWeightInput("2.5")).toBe(2.5); + expect(parseWeightInput("0")).toBe(0); + expect(parseWeightInput(" ")).toBeNull(); + expect(parseWeightInput("-1")).toBeUndefined(); + expect(parseWeightInput("abc")).toBeUndefined(); + expect(parseWeightInput("1.")).toBe(1); + expect(parseWeightInput(".")).toBeUndefined(); + expect(parseWeightInput("1e3")).toBeUndefined(); + expect(parseWeightInput("0x10")).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/routerWeights.ts b/ui/litellm-dashboard/src/components/common_components/routerWeights.ts new file mode 100644 index 00000000000..16f428cade8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerWeights.ts @@ -0,0 +1,105 @@ +export type GroupWeights = Readonly>; +export type RouterWeights = Readonly>; + +export interface WeightableDeployment { + model_name?: string | null; + litellm_params?: { model?: string | null } | null; + model_info?: { id?: string | null; team_id?: string | null; team_public_model_name?: string | null } | null; +} + +export interface WeightRow { + deploymentId: string; + label: string | null; + weight: number | null; + sharePercent: number | null; +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const isWeight = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value) && value >= 0; + +export const routerWeightsFrom = (stored: unknown): RouterWeights => + isRecord(stored) + ? Object.fromEntries( + Object.entries(stored) + .filter((entry): entry is [string, Record] => isRecord(entry[1])) + .map(([group, weights]) => [ + group, + Object.fromEntries( + Object.entries(weights).filter((entry): entry is [string, number] => isWeight(entry[1])), + ), + ]), + ) + : {}; + +const deploymentGroup = (deployment: WeightableDeployment): string | null => + (deployment.model_info?.team_id ? deployment.model_info.team_public_model_name : null) || + deployment.model_name || + null; + +const isWeightable = (deployment: WeightableDeployment, group: string, teamId: string | null): boolean => { + const ownerTeamId = deployment.model_info?.team_id ?? null; + const visibleToCaller = ownerTeamId === null || ownerTeamId === teamId; + return deploymentGroup(deployment) === group && Boolean(deployment.model_info?.id) && visibleToCaller; +}; + +export const withGroup = (weights: RouterWeights, group: string): RouterWeights => + group in weights ? weights : { ...weights, [group]: {} }; + +export const withoutGroup = (weights: RouterWeights, group: string): RouterWeights => + Object.fromEntries(Object.entries(weights).filter(([name]) => name !== group)); + +export const withDeploymentWeight = ( + weights: RouterWeights, + group: string, + deploymentId: string, + weight: number | null, +): RouterWeights => { + const others = Object.fromEntries(Object.entries(weights[group] ?? {}).filter(([id]) => id !== deploymentId)); + return { ...weights, [group]: weight === null ? others : { ...others, [deploymentId]: weight } }; +}; + +export const savedRouterWeights = (weights: RouterWeights): RouterWeights | null => { + const nonEmpty = Object.entries(weights).filter(([, group]) => Object.keys(group).length > 0); + return nonEmpty.length > 0 ? Object.fromEntries(nonEmpty) : null; +}; + +export const hasNoPositiveWeight = (group: GroupWeights): boolean => + Object.keys(group).length > 0 && !Object.values(group).some((weight) => weight > 0); + +export const weightRows = ( + group: string, + groupWeights: GroupWeights, + deployments: readonly WeightableDeployment[], + teamId: string | null, +): WeightRow[] => { + const total = Object.values(groupWeights).reduce((sum, weight) => sum + weight, 0); + const row = (deploymentId: string, label: string | null): WeightRow => { + const weight = groupWeights[deploymentId] ?? null; + return { + deploymentId, + label, + weight, + sharePercent: total > 0 ? Math.round(((weight ?? 0) / total) * 100) : null, + }; + }; + const current = deployments + .filter((deployment) => isWeightable(deployment, group, teamId)) + .map((deployment) => row(deployment.model_info!.id!, deployment.litellm_params?.model ?? null)); + const currentIds = new Set(current.map(({ deploymentId }) => deploymentId)); + const stale = Object.keys(groupWeights) + .filter((deploymentId) => !currentIds.has(deploymentId)) + .map((deploymentId) => row(deploymentId, null)); + return [...current, ...stale]; +}; + +const WEIGHT_DRAFT = /^(\d+\.?\d*|\.\d+)?$/; + +export const parseWeightInput = (raw: string): number | null | undefined => { + const trimmed = raw.trim(); + if (!WEIGHT_DRAFT.test(trimmed)) return undefined; + if (trimmed === "") return null; + const weight = Number(trimmed); + return isWeight(weight) ? weight : undefined; +}; diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index 0fbe043c0d1..bbc849167cc 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -173,6 +173,7 @@ const ROUTER_SETTINGS_DEFAULT = { model_group_alias: null, enable_tag_filtering: false, routing_strategy_args: null, + weights: null, }; const SECTION_PAYLOAD_ADDITIONS: Record> = { diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index ce780a221fd..dee936c4be0 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1784,6 +1784,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp key={routerSettingsKey} ref={routerSettingsRef} accessToken={accessToken || ""} + teamId={selectedCreateKeyTeam?.team_id} value={routerSettings || undefined} onChange={setRouterSettings} modelData={ diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index c6b485a57a1..3d67bed5d93 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -114,6 +114,7 @@ import SearchToolSelector from "../search_tools/SearchToolSelector"; import SkillSelector from "../skills/SkillSelector"; import EditLoggingSettings from "./EditLoggingSettings"; import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion"; +import { RouterWeightsSummary } from "../common_components/RouterSettingsSummary"; import { StorePromptsToggle } from "../common_components/StorePromptsToggle"; import MemberModal from "./EditMembership"; import MemberPermissions from "./member_permissions"; @@ -2673,6 +2674,7 @@ const TeamInfoView: React.FC = ({ {info.router_settings.enable_tag_filtering && (
{t("teams.teamInfo.settingsView.router.tagFilteringEnabled")}
)} + ) : (
{t("teams.teamInfo.settingsView.router.none")}
diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index d4ed30d51b2..06b3e4d6b19 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -20380,5 +20380,21 @@ "query": "query", "request": "request" } + }, + "routerWeights": { + "title": "Deployment Weights", + "description": "Split a model group's traffic across its deployments. A deployment with no weight gets no traffic once any weight in its group is set. Weights only apply with the simple-shuffle routing strategy.", + "ignoredForStrategy": "Weights are ignored with {{strategy}}. Select simple-shuffle to use them.", + "addGroup": "Add a model group", + "addGroupPlaceholder": "Select a model group to weight", + "noModels": "No model groups found", + "noDeployments": "No deployments found for this model group", + "unknownDeployment": "Deployment no longer exists", + "weightFor": "Weight for {{deployment}}", + "share": "{{percent}}% of traffic", + "noPositiveWeight": "Set at least one weight above 0, or remove this model group.", + "removeGroup": "Remove weights for {{group}}", + "summary": "Deployment Weights:", + "loadFailed": "Could not load this model group's deployments. Saved weights are kept." } } From b603e84c020cbfb554d4857dd81bde0a0572e0dc Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 7 Nov 2026 13:33:27 -0800 Subject: [PATCH 15/17] fix(mcp): derive MCP bridge and session keys with the FIPS provider's KBKDF (#50032) * fix(mcp): derive MCP bridge and session keys with the FIPS provider's KBKDF With OpenSSL enforcing FIPS, hashlib.scrypt raises, so every MCP session token and bridge envelope request failed with a 500. Under FIPS the keys now come from LITELLM_SALT_KEY through SP 800-108r1 KBKDF run by the provider's EVP_KDF. Without FIPS the scrypt keys are unchanged * fix(mcp): keep master key rotation invalidating MCP tokens under FIPS Pass the master key as the KBKDF context, so rotating it changes the three FIPS-mode keys as it changes the scrypt keys * test(mcp): annotate new FIPS key test locals with Final * fix(mcp): derive the FIPS MCP keys only when general_settings.fips_mode is on On a FIPS OpenSSL 1.1.1 host (RHEL 8) scrypt still works, so keying the switch on openssl_enforces_fips() sent those hosts to the OpenSSL 3 only KBKDF and broke MCP sessions and bridge envelopes that worked on main. fips_mode_active() is the same reader B1 adds, set by enforce_fips_mode at startup. * refactor(fips): move the salt key KBKDF into encrypt_decrypt_utils The KBKDF helper derives keys from LITELLM_SALT_KEY, so it now sits next to the other salt key helpers, and fips.py goes back to main's version. * Pin the MCP key KBKDF fetch to the FIPS provider Fetch KBKDF with the property query fips=yes, so a change to the process's default properties after boot can't move the derivation to a non-FIPS provider. The shared fips_openssl test fixture drops the pin, since the test OpenSSL has no FIPS provider. --- .../common_utils/encrypt_decrypt_utils.py | 69 +++++++++++++++++++ .../bridge_credentials.py | 10 +++ .../session_credentials.py | 9 +++ .../test_encrypt_decrypt_utils.py | 30 ++++++++ tests/unit/proxy/conftest.py | 1 + .../outbound_credentials/conftest.py | 20 ++++++ .../test_bridge_credentials.py | 31 +++++++++ .../test_session_credentials.py | 63 +++++++++++++++++ 8 files changed, 233 insertions(+) create mode 100644 tests/unit/proxy/mcp_server/outbound_credentials/conftest.py diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index e5f6bc5dda8..644a35ef4ef 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -1,11 +1,14 @@ import base64 +import ctypes import os +import re from collections.abc import Mapping from typing import Final, Literal, cast from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR # Versioned ciphertext marker for AES-256-GCM values. # Format: "v2:gcm:" + base64url(nonce(12) || ciphertext || tag(16)). @@ -42,6 +45,72 @@ def get_salt_key() -> str | None: _get_salt_key: Final = get_salt_key +_SALT_KEY_HEX: Final = re.compile(r"(?:sk-)?((?:[0-9a-fA-F]{2}){32,})") +_OSSL_PARAM_UTF8_STRING: Final = 4 +_OSSL_PARAM_OCTET_STRING: Final = 5 +_OSSL_PARAM_UNMODIFIED: Final = ctypes.c_size_t(-1).value +_DERIVED_KEY_BYTES: Final = 32 +_KBKDF_PROPERTIES: Final = b"fips=yes" + + +class _OsslParam(ctypes.Structure): + _fields_ = ( + ("key", ctypes.c_char_p), + ("data_type", ctypes.c_uint), + ("data", ctypes.c_char_p), + ("data_size", ctypes.c_size_t), + ("return_size", ctypes.c_size_t), + ) + + +def _param(name: bytes, data_type: int, data: bytes) -> _OsslParam: + return _OsslParam(name, data_type, data, len(data), _OSSL_PARAM_UNMODIFIED) + + +def _kbkdf_hmac_sha256(key: bytes, label: bytes, context: bytes) -> bytes: + import _hashlib + + libcrypto: Final = ctypes.CDLL(_hashlib.__file__) + kdf_fetch: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p)( + ("EVP_KDF_fetch", libcrypto) + ) + ctx_new: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(("EVP_KDF_CTX_new", libcrypto)) + derive: Final = ctypes.CFUNCTYPE( + ctypes.c_int, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(_OsslParam) + )(("EVP_KDF_derive", libcrypto)) + ctx_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_CTX_free", libcrypto)) + kdf_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_free", libcrypto)) + params: Final = (_OsslParam * 6)( + _param(b"mac", _OSSL_PARAM_UTF8_STRING, b"HMAC"), + _param(b"digest", _OSSL_PARAM_UTF8_STRING, b"SHA256"), + _param(b"key", _OSSL_PARAM_OCTET_STRING, key), + _param(b"salt", _OSSL_PARAM_OCTET_STRING, label), + _param(b"info", _OSSL_PARAM_OCTET_STRING, context), + ) + derived: Final = ctypes.create_string_buffer(_DERIVED_KEY_BYTES) + kdf: Final = kdf_fetch(None, b"KBKDF", _KBKDF_PROPERTIES) + ctx: Final = ctx_new(kdf) + try: + if not ctx or derive(ctx, derived, _DERIVED_KEY_BYTES, params) != 1: + raise ValueError("OpenSSL could not run KBKDF") + return derived.raw + finally: + ctx_free(ctx) + kdf_free(kdf) + + +def derive_key_from_salt_key(label: bytes, context: bytes) -> str: + """Hex of 32 bytes derived from LITELLM_SALT_KEY under label and context by SP 800-108r1 KBKDF in counter mode + with HMAC-SHA256, run by EVP_KDF from the FIPS provider of the OpenSSL that hashlib uses. Raises ValueError when + LITELLM_SALT_KEY is not at least 32 bytes of hex, optionally prefixed with sk-, or when no FIPS provider runs it.""" + salt_key: Final = _SALT_KEY_HEX.fullmatch(os.getenv(SALT_KEY_ENV_VAR, "")) + if salt_key is None: + raise ValueError( + f"With general_settings.fips_mode on, {SALT_KEY_ENV_VAR} must be at least 32 bytes of hex, such as " + "sk-$(openssl rand -hex 32)" + ) + return _kbkdf_hmac_sha256(bytes.fromhex(salt_key.group(1)), label, context).hex() + def _get_encryption_algorithm() -> str: """ diff --git a/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py index 2326bcebdc3..cc3e99263c9 100644 --- a/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py @@ -16,6 +16,8 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, SecretStr +from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.mcp_server.outbound_credentials.envelope import ( EnvelopeIdentity, EnvelopeKeys, @@ -61,7 +63,15 @@ def envelope_keys_from_master_key(master_key: str) -> EnvelopeKeys: fixed for a process), so the KDF runs once per key and adds nothing to the per-request admission path. The derivation is deterministic; rotating ``master_key`` invalidates every outstanding envelope, which is the intended behavior for a signing-key change. + + When general_settings.fips_mode is on, both keys come from :func:`derive_key_from_salt_key` with the master key as + context instead. """ + if fips_mode_active(): + return EnvelopeKeys( + signing_key=SecretStr(derive_key_from_salt_key(_SIGNING_KEY_DOMAIN, master_key.encode())), + encryption_key=SecretStr(derive_key_from_salt_key(_ENCRYPTION_KEY_DOMAIN, master_key.encode())), + ) signing: Final = hashlib.scrypt( master_key.encode(), salt=_SIGNING_KEY_DOMAIN, diff --git a/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py index 6e3d09f4dd2..7cb89e53cdb 100644 --- a/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py @@ -22,6 +22,8 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, Field, SecretStr, ValidationError +from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.mcp_server.outbound_credentials.session_token import ( AsymmetricSessionKeys, OpenedSessionToken, @@ -59,7 +61,14 @@ def session_keys_from_master_key(master_key: str) -> SessionKeys: into the other. The result is cached (the master key is fixed for a process); rotating ``master_key`` invalidates every outstanding session, which is the intended behavior for a signing-key change. + + When general_settings.fips_mode is on, the key comes from :func:`derive_key_from_salt_key` with the master key as + context instead. """ + if fips_mode_active(): + return SessionKeys( + signing_key=SecretStr(derive_key_from_salt_key(_SESSION_SIGNING_KEY_DOMAIN, master_key.encode())) + ) signing: Final = hashlib.scrypt( master_key.encode(), salt=_SESSION_SIGNING_KEY_DOMAIN, diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index 1163090c510..5b0d5f014f8 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -6,8 +6,11 @@ gate, and the backward-compatibility guarantees that let legacy XSalsa20-Poly130 (nacl) ciphertext and new AES values coexist and decrypt correctly. """ +import _hashlib import base64 +import hmac import re +from typing import Final import pytest @@ -17,6 +20,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_bearer_token, decrypt_if_encrypted_with, decrypt_value_helper, + derive_key_from_salt_key, encrypt_bearer_token, encrypt_value, encrypt_value_helper, @@ -266,3 +270,29 @@ def test_bearer_token_uses_only_header_safe_characters(length: int): assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length + + +_needs_shared_openssl: Final = pytest.mark.skipif( + not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach" +) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.parametrize("salt_key", ["sk-" + bytes(range(32)).hex(), bytes(range(32)).hex().upper()]) +def test_derive_key_from_salt_key_is_sp800_108_counter_mode_hmac_sha256( + monkeypatch: pytest.MonkeyPatch, salt_key: str +) -> None: + label: Final = b"litellm-test:label:" + context: Final = b"sk-test-master-key" + monkeypatch.setenv("LITELLM_SALT_KEY", salt_key) + fixed_input: Final = (1).to_bytes(4, "big") + label + b"\x00" + context + (256).to_bytes(4, "big") + assert derive_key_from_salt_key(label, context) == hmac.digest(bytes(range(32)), fixed_input, "sha256").hex() + + +@_needs_shared_openssl +@pytest.mark.skipif(_hashlib.get_fips_mode() == 1, reason="this OpenSSL runs a FIPS provider") +def test_derive_key_from_salt_key_refuses_an_openssl_without_a_fips_provider(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + bytes(range(32)).hex()) + with pytest.raises(ValueError, match="OpenSSL could not run KBKDF"): + derive_key_from_salt_key(b"litellm-test:label:", b"sk-test-master-key") diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index d08a98242ef..8d8781be0a8 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -523,6 +523,7 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]: def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported"))) + monkeypatch.setattr("litellm.proxy.common_utils.encrypt_decrypt_utils._KBKDF_PROPERTIES", None) @pytest.fixture diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py b/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py new file mode 100644 index 00000000000..bd80cba1c62 --- /dev/null +++ b/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py @@ -0,0 +1,20 @@ +from collections.abc import Iterator + +import pytest + +from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import envelope_keys_from_master_key +from litellm.proxy.mcp_server.outbound_credentials.session_credentials import session_keys_from_master_key + + +@pytest.fixture +def openssl_enforcing_fips(fips_openssl: None) -> Iterator[None]: + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + yield + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + + +@pytest.fixture +def fips_mode_on(openssl_enforcing_fips: None, fips_mode: None) -> None: + return None diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py index 47b6c3c2fa9..0ba19d13955 100644 --- a/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py @@ -9,6 +9,7 @@ through the consumer; and no path leaks the upstream token in a repr. """ from datetime import datetime, timedelta, timezone +from typing import Final import pytest from pydantic import SecretStr @@ -318,3 +319,33 @@ def test_is_bridge_envelope_shaped_rejects_non_envelope_bearer(): assert is_bridge_envelope_shaped("Bearer sk-some-litellm-key") is False assert is_bridge_envelope_shaped("plain-upstream-token") is False assert is_bridge_envelope_shaped("") is False + + +def test_envelope_keys_without_fips_stay_the_scrypt_keys_of_the_master_key() -> None: + keys: Final = envelope_keys_from_master_key(_MASTER_KEY) + assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == ( + "34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed", + "eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250", + ) + + +@pytest.mark.usefixtures("openssl_enforcing_fips") +def test_envelope_keys_on_a_fips_openssl_without_fips_mode_stay_the_scrypt_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + keys: Final = envelope_keys_from_master_key(_MASTER_KEY) + assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == ( + "34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed", + "eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250", + ) + + +@pytest.mark.usefixtures("fips_mode_on") +@pytest.mark.parametrize("salt_key", ["", "sk-salt-change-me", _MASTER_KEY, "sk-" + "5a" * 31]) +def test_envelope_keys_under_fips_refuse_a_salt_key_that_is_not_32_bytes_of_hex( + monkeypatch: pytest.MonkeyPatch, salt_key: str +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", salt_key) + with pytest.raises(ValueError, match="LITELLM_SALT_KEY"): + envelope_keys_from_master_key(_MASTER_KEY) diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py index f084d5c7d1d..795ded8528c 100644 --- a/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py @@ -1,6 +1,8 @@ """Tests for the session-token KDF and the edge/token-endpoint resolvers.""" +import _hashlib from datetime import datetime, timedelta, timezone +from typing import Final import pytest from cryptography.hazmat.primitives import serialization @@ -230,3 +232,64 @@ def test_signing_config_error_detail_never_leaks_key_material(): ) assert isinstance(resolved, SessionSigningConfigError) assert pem.splitlines()[1] not in resolved.detail + + +def test_session_key_without_fips_stays_the_scrypt_key_of_the_master_key() -> None: + keys: Final = session_keys_from_master_key(MASTER_KEY) + assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d" + + +@pytest.mark.usefixtures("openssl_enforcing_fips") +def test_session_key_on_a_fips_openssl_without_fips_mode_stays_the_scrypt_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + keys: Final = session_keys_from_master_key(MASTER_KEY) + assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d" + + +def _fips_keys(master_key: str) -> tuple[str, str, str]: + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + envelope: Final = envelope_keys_from_master_key(master_key) + session: Final = session_keys_from_master_key(master_key) + return ( + envelope.signing_key.get_secret_value(), + envelope.encryption_key.get_secret_value(), + session.signing_key.get_secret_value(), + ) + + +_needs_shared_openssl: Final = pytest.mark.skipif( + not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach" +) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_mode_on") +def test_keys_under_fips_come_from_the_salt_key_are_distinct_and_the_same_on_every_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32) + first: Final = _fips_keys(MASTER_KEY) + assert first == _fips_keys(MASTER_KEY) + assert len(set(first)) == 3, first + assert first == ( + "ef694d415772aef09731c47709d4d9da47f7bbfcd6484836dea09c2bb387063b", + "d8a7cd70690fbf98c5581301b2e0460f2ad714f7c74b2af1b3dac3e0aa99db85", + "5e11e470bf9c838165b70d0b2b82a7ee5e2f00d9449eeb54f7c5bf9491b64354", + ) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_mode_on") +def test_keys_under_fips_all_change_when_the_master_key_rotates(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32) + before: Final = _fips_keys(MASTER_KEY) + after: Final = _fips_keys("sk-a-different-master-key") + assert set(before).isdisjoint(after), (before, after) + + +@pytest.mark.usefixtures("fips_mode_on") +def test_session_key_under_fips_refuses_an_unset_salt_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + with pytest.raises(ValueError, match="LITELLM_SALT_KEY"): + session_keys_from_master_key(MASTER_KEY) From fa254b93467a35204729a1fea078b23d584ba71a Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 7 Nov 2026 14:53:07 -0800 Subject: [PATCH 16/17] fix(responses): surface mid-stream upstream errors that arrive without an event type (#50466) * fix(responses): surface mid-stream upstream errors that arrive without an event type Bedrock's native Responses route can end a stream with a bare {"error": {...}} chunk. It parsed as a typeless GenericEvent and crashed the iterator with AttributeError, so clients saw that instead of the upstream error. Such chunks are now read as error events, which raise the mapped upstream error and reach the client as response.failed. * refactor(responses): inline the untyped error chunk check to keep LIT001 at the base count --- litellm/responses/streaming_iterator.py | 11 ++- .../test_bedrock_gpt_responses_native_wire.py | 61 +++++++++++++++- .../test_streaming_iterator_error_events.py | 70 +++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 4273647567e..08b2c518632 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -486,9 +486,18 @@ class BaseResponsesAPIStreamingIterator: if _is_json_object(parsed_chunk): if self.responses_api_provider_config is None: raise ValueError("responses_api_provider_config is required to process live streaming chunks") + typed_chunk: Final = ( + parsed_chunk + if parsed_chunk.get("type") is not None or "error" not in parsed_chunk + else { + "sequence_number": SEQUENCE_NUMBER_STAMPED_ON_EMISSION, + **parsed_chunk, + "type": ResponsesAPIStreamEvents.ERROR.value, + } + ) openai_responses_api_chunk: Final = self.responses_api_provider_config.transform_streaming_response( model=self.model, - parsed_chunk=parsed_chunk, + parsed_chunk=typed_chunk, logging_obj=self.logging_obj, ) diff --git a/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py b/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py index 159097c80ee..a5cba28a48a 100644 --- a/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py +++ b/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py @@ -1,14 +1,15 @@ import base64 +import json import uuid from dataclasses import dataclass from typing import Final import openai -from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of +from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, marker_of, respond, target_of from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows -from integration._support.wire import Request, Wire, wire_server -from openai.types.responses import ResponseCompletedEvent, ResponseTextDeltaEvent +from integration._support.wire import Reply, Request, Wire, wire_server +from openai.types.responses import ResponseCompletedEvent, ResponseFailedEvent, ResponseTextDeltaEvent from pydantic import JsonValue, TypeAdapter from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with @@ -17,6 +18,7 @@ GPT: Final = "us.openai.gpt-5.6-sol" TOKEN: Final = "synthetic-bedrock-bearer" SALT: Final = "sk-integration-salt" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +UPSTREAM_OUTAGE: Final = "The server had an error while processing your request. Sorry about that!" @dataclass(frozen=True, slots=True) @@ -123,6 +125,59 @@ async def test_async_openai_sdk_responses_stream_is_served_by_the_native_respons assert _spend_row(completed.response.id) == _success_row(model) +def _stream_then_bare_error(request: Request) -> Reply: + marker: Final = marker_of(request) + created: Final[dict[str, JsonValue]] = { + "type": "response.created", + "sequence_number": 0, + "response": { + "id": f"resp_upstream_{marker}", + "object": "response", + "created_at": 1, + "status": "in_progress", + "model": GPT, + "output": [], + }, + } + delta: Final[dict[str, JsonValue]] = { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{marker}", + "output_index": 0, + "content_index": 0, + "delta": answer(marker), + } + outage: Final[dict[str, JsonValue]] = { + "error": {"message": UPSTREAM_OUTAGE, "type": "server_error", "code": "service_unavailable"} + } + return Reply( + content_type="text/event-stream", + chunks=tuple(f"data: {json.dumps(event)}\n\n".encode() for event in (created, delta, outage)), + ) + + +async def test_mid_stream_upstream_error_without_an_event_type_reaches_the_client_as_response_failed( + gateway: Gateway, +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_stream_then_bare_error) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = openai.AsyncOpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0) + stream: Final = await client.responses.create( + model=model, input=_prompt(marker), stream=True, extra_body={"cache": {"no-cache": True}} + ) + events: Final = [event async for event in stream] + assert [event.type for event in events] == [ + "response.created", + "response.output_text.delta", + "response.failed", + ], events + failed: Final = events[-1] + assert isinstance(failed, ResponseFailedEvent), failed + assert failed.response.error is not None and UPSTREAM_OUTAGE in failed.response.error.message, failed + _native_request(wire) + + def test_codex_agent_message_with_encrypted_content_reaches_the_native_responses_route_intact( gateway: Gateway, ) -> None: diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index 0acf5d9ce62..29cb0412422 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -959,3 +959,73 @@ async def test_responses_websocket_turn_failure_reports_the_deployment_it_was_se assert failure is not None _assert_names_the_azure_deployment(failure, reported_api_base) + + +_BEDROCK_SERVICE_UNAVAILABLE_CHUNK: Final = { + "error": { + "message": "The server had an error while processing your request. Sorry about that!", + "type": "server_error", + "code": "service_unavailable", + } +} + + +def _real_config_iterator(provider: str, model: str, events: tuple[Mapping[str, object], ...]): + payload: Final = b"".join(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events) + + async def aiter_bytes(): + yield payload + + response: Final = Mock() + response.headers = {} + response.aiter_bytes = aiter_bytes + logging_obj: Final = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"litellm_params": {}} + logging_obj.completion_start_time = None + config: Final = ProviderConfigManager.get_provider_responses_api_config( + provider=litellm.LlmProviders(provider), model=model + ) + assert config is not None + return ResponsesAPIStreamingIterator( + response=response, + model=model, + responses_api_provider_config=config, + logging_obj=logging_obj, + custom_llm_provider=provider, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("provider", "model"), + [ + ("bedrock", "us.openai.gpt-5.6-sol"), + ("openai", "gpt-5.5"), + ("volcengine", "doubao-seed-1-6"), + ("bigmodel", "glm-5"), + ], +) +async def test_untyped_mid_stream_error_chunk_raises_the_upstream_error(provider: str, model: str): + iterator: Final = _real_config_iterator( + provider, + model, + ( + { + "type": "response.output_text.delta", + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": "partial ", + "sequence_number": 1, + }, + _BEDROCK_SERVICE_UNAVAILABLE_CHUNK, + ), + ) + stream: Final = aiter(iterator) + assert (await anext(stream)).type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA + + with pytest.raises(MidStreamFallbackError) as exc_info: + await anext(stream) + assert "The server had an error while processing your request" in str(exc_info.value.original_exception) + assert exc_info.value.status_code >= 500 + assert exc_info.value.generated_content == "partial " From 0d0bf8f75f63a9c76ab75d121cdd6b7f498f2607 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:08:23 -0800 Subject: [PATCH 17/17] test(integration): give each kill-burst its own prompts so cached control replies cannot stand in for late traffic (#50569) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend/test_normalized_error_long_message.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/spend/test_normalized_error_long_message.py b/tests/integration/spend/test_normalized_error_long_message.py index 23200c0450e..23769391427 100644 --- a/tests/integration/spend/test_normalized_error_long_message.py +++ b/tests/integration/spend/test_normalized_error_long_message.py @@ -358,16 +358,17 @@ def _burst( *, rejected_model: str = CRAFTED_MODEL, ) -> tuple[_BurstResult, ...]: + nonce: Final = uuid.uuid4().hex crafted: Final = tuple( (f"crafted-{path}-{index}", path, _body(path, rejected_model, uuid.uuid4().hex, stream=index % 2 == 1), open_key) for path in (CHAT, MESSAGES, RESPONSES) for index in range(4) ) happy: Final = tuple( - (f"happy-{index}", CHAT, _body(CHAT, happy_model, f"happy-{index}"), happy_key) for index in range(8) + (f"happy-{index}", CHAT, _body(CHAT, happy_model, f"happy-{index}-{nonce}"), happy_key) for index in range(8) ) late: Final = tuple( - (f"late-{index}", CHAT, _body(CHAT, happy_model, f"late-{index}"), happy_key) for index in range(8) + (f"late-{index}", CHAT, _body(CHAT, happy_model, f"late-{index}-{nonce}"), happy_key) for index in range(8) ) with ( ThreadPoolExecutor(max_workers=28) as pool, @@ -427,6 +428,7 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy open_key: Final = scenario.key() control: Final = _burst(client, model, key, open_key, lambda: None, rejected_model=BENIGN_288KB_MODEL) assert all(result.status == (400 if result.label.startswith("crafted") else 200) for result in control), control + wire.drain() workers: Final = owned.worker_pids() assert len(workers) == 2, workers victim: Final = min(workers) @@ -438,6 +440,8 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy _assert_burst_rejections_fast(results, control) late: Final = tuple(result for result in results if result.label.startswith("late")) assert all(result.status == 200 for result in late), late + seen: Final = tuple(request.body.decode() for request in wire.drain()) + assert all(any(f"normalized error audit {result.label}-" in body for body in seen) for result in late), seen _assert_rows_land_exactly_once(results, "late") survivor: Final = tuple(pid for pid in owned.worker_pids() if pid != victim) assert survivor, "no worker left serving"