diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 70c2d600ba3..2177e483a1b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -336,9 +336,9 @@ Each translation is isolated in its own file, making it easy to test and modify | `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` | | `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` | | `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` | -| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` | +| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/messages/native/transformation.py` | | `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py` | -| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` | +| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py` | | Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` | ### Example: Debugging prompt caching diff --git a/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs b/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs index a4aee4eb4da..cfa9e53aa7b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/mapping.rs @@ -235,7 +235,7 @@ python_callbacks! { "async_log_stream_event", "async_post_mcp_tool_call_hook", ], - Dispatch::Python("litellm.llms.anthropic.pass_through.messages.handler") => [ + Dispatch::Python("litellm.llms.anthropic.messages.native.handler") => [ "async_pre_request_hook", ], Dispatch::Python("litellm.litellm_core_utils.streaming_handler") => [ diff --git a/litellm/__init__.py b/litellm/__init__.py index 117ba9604e9..c74292c5888 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1729,6 +1729,9 @@ if TYPE_CHECKING: from .llms.hosted_vllm.decisions.transformation import ( HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig, ) + from .llms.lm_studio.decisions.transformation import ( + LMStudioDecisionsConfig as LMStudioDecisionsConfig, + ) from .llms.laya.decisions.transformation import ( LayaDecisionsConfig as LayaDecisionsConfig, ) @@ -1738,6 +1741,9 @@ if TYPE_CHECKING: from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) + from .llms.azure.decisions.transformation import ( + AzureDecisionsConfig as AzureDecisionsConfig, + ) from .llms.azure_ai.decisions.transformation import ( AzureAIDecisionsConfig as AzureAIDecisionsConfig, ) @@ -1782,7 +1788,7 @@ if TYPE_CHECKING: SagemakerNovaConfig as SagemakerNovaConfig, ) from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig - from .llms.anthropic.pass_through.messages.transformation import ( + from .llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig as AnthropicMessagesConfig, ) from .llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index a56004d1274..f43e369e092 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -170,9 +170,11 @@ LLM_CONFIG_NAMES: Final = ( "StrandsDeciderDecisionsConfig", "DatabricksDecisionsConfig", "HostedVLLMDecisionsConfig", + "LMStudioDecisionsConfig", "LayaDecisionsConfig", "BespokeDecisionsConfig", "OpenAIDecisionsConfig", + "AzureDecisionsConfig", "AzureAIDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", @@ -746,9 +748,11 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.hosted_vllm.decisions.transformation", "HostedVLLMDecisionsConfig", ), + "LMStudioDecisionsConfig": (".llms.lm_studio.decisions.transformation", "LMStudioDecisionsConfig"), "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"), "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), + "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"), "AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", @@ -797,7 +801,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ), "CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"), "AnthropicMessagesConfig": ( - ".llms.anthropic.pass_through.messages.transformation", + ".llms.anthropic.messages.native.transformation", "AnthropicMessagesConfig", ), "BedrockClaudePlatformMessagesConfig": ( diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 7089ff78835..affabc6cc74 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -56,7 +56,7 @@ from litellm.types.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( AnthropicMessagesStreamCacheWriter, ) from litellm.types.utils import PromptTokensDetailsWrapper @@ -182,7 +182,7 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo spend and callback records. A plain (non-stream) replay logs here, since nothing else will. """ - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( CachedAnthropicMessagesStreamIterator, ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -1040,7 +1040,7 @@ class LLMCachingHandler: elif ( call_type == CallTypes.anthropic_messages.value or call_type == CallTypes.aanthropic_messages.value ) and isinstance(cached_result, dict): - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( convert_cached_anthropic_messages_result, ) @@ -1270,7 +1270,7 @@ class LLMCachingHandler: return result if not isinstance(result, AsyncIterator): return result - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( AnthropicMessagesStreamCacheWriter, ) diff --git a/litellm/images/main.py b/litellm/images/main.py index 29bac3ffe05..9c88e737a40 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars import importlib -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from functools import partial from typing import TYPE_CHECKING, Final, Literal, cast, overload @@ -85,6 +85,47 @@ def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]: return _ImageEditRequestUtils_cache +def _chatgpt_image_params(kwargs: Mapping[str, object]) -> GenericLiteLLMParams: + from litellm.responses.utils import ResponsesAPIRequestUtils + + headers: Final = ResponsesAPIRequestUtils.merge_client_forwarded_headers( + TypeAdapter[dict[str, object] | None](dict[str, object] | None).validate_python(kwargs.get("extra_headers")), + TypeAdapter[dict[str, str] | None](dict[str, str] | None).validate_python(kwargs.get("headers")), + ) + return GenericLiteLLMParams.model_validate({**kwargs, "extra_headers": headers}) + + +def _chatgpt_image_headers(model: str, params: GenericLiteLLMParams, headers: Mapping[str, str]) -> Mapping[str, str]: + from litellm.llms.chatgpt.common_utils import get_chatgpt_request_credential + from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig + + credential: Final = get_chatgpt_request_credential(params) + credential_headers: Final = ( + { + "Authorization": f"Bearer {credential.access_token}", + **({"ChatGPT-Account-Id": credential.account_id} if credential.account_id else {}), + } + if credential is not None + else {} + ) + explicit_account_id: Final = next( + (value for name, value in headers.items() if name.lower() == "chatgpt-account-id"), None + ) + auth_headers: Final = ChatGPTResponsesAPIConfig().validate_environment( + headers=credential_headers, model=model, litellm_params=params + ) + return { + **{k: v for k, v in auth_headers.items() if k.lower() not in ("content-type", "accept")}, + **{ + k: v + for k, v in headers.items() + if k.lower() not in ("authorization", "chatgpt-account-id", "cookie", "x-litellm-api-key") + }, + **({"ChatGPT-Account-Id": explicit_account_id} if credential is None and explicit_account_id else {}), + "Accept": "application/json", + } + + ##### Image Generation ####################### @client async def aimage_generation(*args, **kwargs) -> ImageResponse: @@ -121,6 +162,12 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse: model=model, custom_llm_provider=provider_hints.get("custom_llm_provider"), api_base=provider_hints.get("api_base"), + litellm_params=( + _chatgpt_image_params(TypeAdapter(dict[str, object]).validate_python(kwargs)) + if provider_hints.get("custom_llm_provider") == "chatgpt" + or (isinstance(model, str) and model.startswith("chatgpt/")) + else None + ), ) # Await normally @@ -240,12 +287,18 @@ def image_generation( if extra_headers is not None: headers.update(extra_headers) model_response: ImageResponse = litellm.utils.ImageResponse() + chatgpt_params: Final = ( + _chatgpt_image_params(TypeAdapter(dict[str, object]).validate_python({**kwargs, "api_key": api_key})) + if custom_llm_provider == "chatgpt" or (model is not None and model.startswith("chatgpt/")) + else None + ) dynamic_api_key: str | None = None if model is not None or custom_llm_provider is not None: model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, + litellm_params=chatgpt_params, ) else: model = "dall-e-2" @@ -268,7 +321,10 @@ def image_generation( "size", "style", ] - non_default_params: Final = filter_out_litellm_params(kwargs, excluding=openai_params) + non_default_params: Final = filter_out_litellm_params( + kwargs, + excluding=[*openai_params, "extra_headers", "headers"] if chatgpt_params is not None else openai_params, + ) image_generation_config: BaseImageGenerationConfig | None = None if custom_llm_provider is not None and custom_llm_provider in LlmProviders._member_map_.values(): @@ -490,7 +546,7 @@ def image_generation( or custom_llm_provider == LlmProviders.LITELLM_PROXY.value or custom_llm_provider in litellm.openai_compatible_providers ): - if extra_headers is not None: + if extra_headers is not None and chatgpt_params is None: optional_params["extra_headers"] = extra_headers # Forward OpenAI organization if present (set by proxy pre-call utils) organization: Final[str | None] = kwargs.get("organization", None) @@ -506,7 +562,17 @@ def image_generation( organization=organization, aimg_generation=aimg_generation, client=client, - headers=headers, + headers=( + dict( + _chatgpt_image_headers( + model, + chatgpt_params.model_copy(update={"api_key": dynamic_api_key or api_key}), + TypeAdapter(dict[str, str]).validate_python(headers), + ) + ) + if chatgpt_params is not None + else headers + ), litellm_params=litellm_params_dict, ) elif custom_llm_provider == "bedrock": diff --git a/litellm/integrations/otel/model/request_io.py b/litellm/integrations/otel/model/request_io.py index 287d28a0d17..82225531e23 100644 --- a/litellm/integrations/otel/model/request_io.py +++ b/litellm/integrations/otel/model/request_io.py @@ -6,7 +6,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.integrations.otel.mappers.utils import json_or_none -from litellm.llms.anthropic.pass_through.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream +from litellm.llms.anthropic.messages.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse from litellm.types.utils import ModelResponse, ModelResponseStream diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index d0ed30d40dd..3a2d7ee33bb 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -113,7 +113,7 @@ def _chat_request_from_anthropic_messages( because the logged optional_params switch dialect per provider path (the bridge's inner completion rewrites them to chat shape mid-flight); the adapter translates them alongside the messages, and sampling params copy through untranslated.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index 255c1f1adb2..4ba8c2daa63 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -70,7 +70,7 @@ Claude Code (Anthropic's official CLI) sends web search requests using Anthropic Native tools are converted to LiteLLM standard format **before** sending to the provider: -1. **Conversion Point** (`litellm/llms/anthropic/pass_through/messages/handler.py`): +1. **Conversion Point** (`litellm/llms/anthropic/messages/native/handler.py`): - In `anthropic_messages()` function (lines 60-127) - Runs BEFORE the API request is made - Detects native web search tools using `is_web_search_tool()` diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 3bc8dc4e23e..de8c6e6c4cc 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -6354,7 +6354,7 @@ def is_flat_responses_tool(tool: Mapping[str, object]) -> bool: def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) diff --git a/litellm/litellm_core_utils/renamed_package_shim.py b/litellm/litellm_core_utils/renamed_package_shim.py new file mode 100644 index 00000000000..7f7f39a351a --- /dev/null +++ b/litellm/litellm_core_utils/renamed_package_shim.py @@ -0,0 +1,87 @@ +import importlib +import sys +import warnings +from collections.abc import Callable, Sequence +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import ModuleSpec +from importlib.util import spec_from_loader +from types import ModuleType +from typing import Final + + +class _AliasLoader(Loader): + """Hand the import system the module already loaded under its new name instead of executing a copy.""" + + def __init__(self, target: ModuleType) -> None: + self._target: Final = target + self._spec: Final = target.__spec__ + + def create_module(self, spec: ModuleSpec) -> ModuleType: + return self._target + + def exec_module(self, module: ModuleType) -> None: + module.__spec__ = self._spec # rebind-ok: the loader restores the spec the import system replaced + + +class _RenamedPackageFinder(MetaPathFinder): + def __init__(self, old_name: str, renames: tuple[tuple[str, str], ...]) -> None: + self._prefix: Final = old_name + "." + self._renames: Final = renames + + def new_name_for(self, old_module_name: str) -> str | None: + if not old_module_name.startswith(self._prefix): + return None + for old_prefix, new_prefix in self._renames: + if old_module_name == old_prefix or old_module_name.startswith(old_prefix + "."): + return new_prefix + old_module_name[len(old_prefix) :] + return None + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None = None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + new_name: Final = self.new_name_for(fullname) + if new_name is None: + return None + module: Final = importlib.import_module(new_name) + spec: Final = spec_from_loader(fullname, _AliasLoader(module), is_package=hasattr(module, "__path__")) + if spec is not None and spec.submodule_search_locations is not None: + spec.submodule_search_locations = list(module.__path__) # pyright: ignore[reportAny] # module attributes are untyped + return spec + + +def alias_renamed_package( + old_name: str, + new_name: str, + *, + nested_renames: tuple[tuple[str, str], ...] = (), +) -> Callable[[str], ModuleType]: + """Make every module of the package now at ``new_name`` importable under its old dotted name. + + Nothing is imported up front: a meta path finder resolves ``old_name.x.y`` to the module object at + ``new_name.x.y`` the moment something imports it, so a deep import of the old path is the very same object + as the new path and importing one old module loads no more than importing its new counterpart would. + ``nested_renames`` lists (old, new) dotted prefixes for sub-packages that changed name inside the move, + longest first. The returned function is the shim package's module ``__getattr__``, so attribute access on + the old package reaches the renamed sub-packages too. + """ + warnings.warn( + f"{old_name} moved to {new_name}; update imports, the old path will be removed in a future release", + DeprecationWarning, + stacklevel=3, + ) + finder: Final = _RenamedPackageFinder(old_name, (*nested_renames, (old_name, new_name))) + if not any( + isinstance(entry, _RenamedPackageFinder) and entry.new_name_for(old_name + ".x") for entry in sys.meta_path + ): + sys.meta_path.insert(0, finder) + + def getattr_(name: str) -> ModuleType: + try: + return importlib.import_module(f"{old_name}.{name}") + except ModuleNotFoundError as error: + raise AttributeError(f"module {old_name!r} has no attribute {name!r}") from error + + return getattr_ diff --git a/litellm/litellm_core_utils/served_output_texts.py b/litellm/litellm_core_utils/served_output_texts.py index e23db5200a3..854785f304b 100644 --- a/litellm/litellm_core_utils/served_output_texts.py +++ b/litellm/litellm_core_utils/served_output_texts.py @@ -110,7 +110,7 @@ def served_output_texts(response: object) -> ServedTexts | None: def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None: if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks): return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream))) - from litellm.llms.anthropic.pass_through.stream_assembly import ( + from litellm.llms.anthropic.messages.stream_assembly import ( assemble_anthropic_sse_stream, is_anthropic_sse_stream, ) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 90e79d4d853..ee5f70fb394 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -261,6 +261,10 @@ def get_image_dimensions( _header, encoded = data.split(",", 1) img_data = base64.b64decode(encoded) + return _image_dimensions_from_bytes(img_data) + + +def _image_dimensions_from_bytes(img_data: bytes) -> tuple[int, int]: img_type: Final = get_image_type(img_data) if img_type == "png": @@ -307,8 +311,37 @@ def get_image_dimensions( return DEFAULT_IMAGE_WIDTH, DEFAULT_IMAGE_HEIGHT +def _auto_image_dimensions(data: str) -> tuple[int, int] | None: + """Read bounded inline image data without fetching caller-supplied URLs.""" + if not data.startswith("data:image/"): + return None + max_bytes: Final = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024) + max_encoded_length: Final = 4 * ((max_bytes + 2) // 3) + # Bound allocations before splitting or decoding the data URL, including its header. + if max_bytes <= 0 or len(data) > max_encoded_length + 128: + return None + header, separator, encoded = data.partition(",") + if not separator or len(header) > 128 or not header.endswith(";base64") or len(encoded) > max_encoded_length: + return None + try: + image_data: Final = base64.b64decode(encoded, validate=True) + if len(image_data) > max_bytes or get_image_type(image_data) not in ( + "png", + "gif", + "jpeg", + "webp", + ): + return None + width, height = _image_dimensions_from_bytes(image_data) + except (ValueError, TypeError, struct.error): + return None + if width <= 0 or height <= 0: + return None + return width, height + + def calculate_img_tokens( - data, + data: str, mode: Literal["low", "high", "auto"] = "auto", base_tokens: int = 85, # openai default - https://openai.com/pricing use_default_image_token_count: bool = False, @@ -318,7 +351,8 @@ def calculate_img_tokens( Args: data (str): The URL or base64 encoded string of the image. - mode (Literal["low", "high", "auto"]): The mode to use for calculating the number of tokens. + mode (Literal["low", "high", "auto"]): Auto uses high-detail tiles for readable inline images; + URLs and unreadable images keep the base estimate without network I/O. base_tokens (int): The base number of tokens for an image. use_default_image_token_count (bool): When True, will NOT make a GET request to the image URL and instead return the default image dimensions. @@ -328,18 +362,19 @@ def calculate_img_tokens( if use_default_image_token_count: verbose_logger.debug("Using default image token count: %s", DEFAULT_IMAGE_TOKEN_COUNT) return DEFAULT_IMAGE_TOKEN_COUNT - if mode == "low" or mode == "auto": + if mode == "low": return base_tokens - elif mode == "high": - # Run the async function using the helper - width, height = get_image_dimensions( - data=data, - ) - resized_width, resized_height = resize_image_high_res(width=width, height=height) - tiles_needed_high_res = calculate_tiles_needed(resized_width=resized_width, resized_height=resized_height) - tile_tokens: Final = (base_tokens * 2) * tiles_needed_high_res - total_tokens: Final = base_tokens + tile_tokens - return total_tokens + if mode not in ("auto", "high"): + return None + dimensions: Final = _auto_image_dimensions(data) if mode == "auto" else get_image_dimensions(data=data) + if dimensions is None: + return base_tokens + width, height = dimensions + resized_width, resized_height = resize_image_high_res(width=width, height=height) + tiles_needed_high_res = calculate_tiles_needed(resized_width=resized_width, resized_height=resized_height) + tile_tokens: Final = (base_tokens * 2) * tiles_needed_high_res + total_tokens: Final = base_tokens + tile_tokens + return total_tokens TokenCounterFunction = Callable[[str], int] diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 3ab21099d3a..d371aab4581 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -26,18 +26,18 @@ from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, LiteLLMAnthropicMessagesAdapter, is_provider_native_tool_dict, truncate_tool_name, ) -from litellm.llms.anthropic.pass_through.convertible_blocks import ( +from litellm.llms.anthropic.messages.convertible_blocks import ( keep_non_string_tool_result_texts, scannable_tool_result_request, ) -from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details +from litellm.llms.anthropic.messages.stream_assembly import ( build_complete_streaming_response, build_usage_only_response_from_chunks, ) @@ -435,7 +435,7 @@ class AnthropicMessagesHandler(BaseTranslation): def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]: import uuid - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.guardrail_translation.utils import ( diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index c2d87c641be..eb8e595c849 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -162,7 +162,7 @@ def _has_non_chat_user_block(message: Mapping[str, object]) -> bool: def _gets_placeholder_name(tool: Mapping[str, object]) -> bool: - from litellm.llms.anthropic.pass_through.adapters.transformation import is_provider_native_tool_dict + from litellm.llms.anthropic.messages.adapters.transformation import is_provider_native_tool_dict name: Final = tool.get("name") return not (_is_chat_tool(tool) or is_provider_native_tool_dict(tool) or (isinstance(name, str) and name.strip())) diff --git a/litellm/llms/anthropic/pass_through/adapters/__init__.py b/litellm/llms/anthropic/messages/adapters/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/adapters/__init__.py rename to litellm/llms/anthropic/messages/adapters/__init__.py diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py similarity index 98% rename from litellm/llms/anthropic/pass_through/adapters/handler.py rename to litellm/llms/anthropic/messages/adapters/handler.py index 01e781485a0..e5dd1d3c94f 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -15,23 +15,23 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( AnthropicAdapter, ) -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, PolyfillResult, apply_context_management, ) -from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text -from litellm.llms.anthropic.pass_through.safeguards import ( +from litellm.llms.anthropic.messages.convertible_blocks import reject_malformed_tool_result_text +from litellm.llms.anthropic.messages.safeguards import ( SafeguardsEvaluator, build_safeguards_evaluator, read_classifier_model_setting, requested_dangerous_tool_use, with_safeguard_results, ) -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.utils import ( ProxySubCallLoggingKwargs, is_reasoning_auto_summary_enabled, litellm_logging_obj_from_kwargs, @@ -158,7 +158,7 @@ async def prepare_context_managed_request( drop_invalid_edits: bool = False, ) -> PolyfillResult | None: """Apply client compaction history, then optional context_management polyfill.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( apply_client_compaction_block_history, ) @@ -239,7 +239,7 @@ def _polyfill_will_run( if edits is None: return False - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_EDIT_TYPE, ) @@ -265,7 +265,7 @@ def _spec_has_non_compact_edits( if edits is None: return False - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_EDIT_TYPE, ) @@ -298,7 +298,7 @@ def _normalize_spec_edits( if context_management_explicitly_dropped(additional_drop_params): return None - from litellm.llms.anthropic.pass_through.context_management.dispatcher import ( + from litellm.llms.anthropic.messages.context_management.dispatcher import ( normalize_spec, ) @@ -543,7 +543,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: Handles both string ("max") and dict ({"effort": "max", "summary": ...}) formats. Uses model registry to check supports_xhigh/supports_minimal. """ - from litellm.llms.anthropic.pass_through.utils import ( + from litellm.llms.anthropic.messages.utils import ( normalize_reasoning_effort_value, ) diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py similarity index 99% rename from litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py rename to litellm/llms/anthropic/messages/adapters/streaming_iterator.py index 4eb53f35a03..13acb19e454 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -27,8 +27,8 @@ from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import set_hidden_params -from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults -from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame +from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults +from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( AppliedEdit, @@ -46,7 +46,7 @@ from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDe if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject - from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponseStream @@ -234,7 +234,7 @@ class _CombinedChunkSplitter: @staticmethod def _is_combined(chunk: "ModelResponseStream") -> bool: """True if ``chunk`` carries response content AND a finish_reason.""" - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( openai_chat_refusal_text, ) @@ -1308,7 +1308,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): delta: Final = processed_chunk["delta"] if delta.get("stop_reason") == "max_tokens": return processed_chunk - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( refusal_stop_details, ) @@ -1363,7 +1363,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): @staticmethod def _is_blank_delta(chunk: "ModelResponseStream") -> bool: from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( openai_chat_refusal_text, ) @@ -1400,7 +1400,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): - Different content types in the response - Specific markers in the content """ - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( openai_chat_refusal_text, ) diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py similarity index 99% rename from litellm/llms/anthropic/pass_through/adapters/transformation.py rename to litellm/llms/anthropic/messages/adapters/transformation.py index 9b1297dbd75..5ab112d0bfe 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -10,8 +10,8 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.hidden_params import get_hidden_params -from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part +from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, prompt_cache_key_from_user_id, @@ -139,14 +139,14 @@ from litellm.llms.anthropic.common_utils import ( normalize_anthropic_tool_use_id, strip_encrypted_reasoning_blocks_from_anthropic_messages, ) -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( PolyfillResult, ) -from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( +from litellm.llms.anthropic.messages.native.mid_conversation_system import ( convert_mid_conversation_system_turns, is_system_role_message, ) -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( openai_chat_refusal_text, refusal_stop_details, ) @@ -210,7 +210,7 @@ from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject - from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.llms.anthropic import ContentBlockContentBlockDict ToolResultContent: TypeAlias = str | list[ToolMessageContentPart] diff --git a/litellm/llms/anthropic/pass_through/architecture.md b/litellm/llms/anthropic/messages/architecture.md similarity index 100% rename from litellm/llms/anthropic/pass_through/architecture.md rename to litellm/llms/anthropic/messages/architecture.md diff --git a/litellm/llms/anthropic/pass_through/caller_model_checks.py b/litellm/llms/anthropic/messages/caller_model_checks.py similarity index 100% rename from litellm/llms/anthropic/pass_through/caller_model_checks.py rename to litellm/llms/anthropic/messages/caller_model_checks.py diff --git a/litellm/llms/anthropic/pass_through/context_management/__init__.py b/litellm/llms/anthropic/messages/context_management/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/__init__.py rename to litellm/llms/anthropic/messages/context_management/__init__.py diff --git a/litellm/llms/anthropic/pass_through/context_management/constants.py b/litellm/llms/anthropic/messages/context_management/constants.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/constants.py rename to litellm/llms/anthropic/messages/context_management/constants.py diff --git a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py b/litellm/llms/anthropic/messages/context_management/dispatcher.py similarity index 98% rename from litellm/llms/anthropic/pass_through/context_management/dispatcher.py rename to litellm/llms/anthropic/messages/context_management/dispatcher.py index bce653df6d1..1557cf206ac 100644 --- a/litellm/llms/anthropic/pass_through/context_management/dispatcher.py +++ b/litellm/llms/anthropic/messages/context_management/dispatcher.py @@ -18,7 +18,7 @@ from .editors import ( from .result import PolyfillResult if TYPE_CHECKING: - from litellm.llms.anthropic.pass_through.utils import ProxySubCallLoggingKwargs + from litellm.llms.anthropic.messages.utils import ProxySubCallLoggingKwargs from litellm.router import Router from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/__init__.py b/litellm/llms/anthropic/messages/context_management/editors/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/editors/__init__.py rename to litellm/llms/anthropic/messages/context_management/editors/__init__.py diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py b/litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py rename to litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py diff --git a/litellm/llms/anthropic/pass_through/context_management/editors/compact.py b/litellm/llms/anthropic/messages/context_management/editors/compact.py similarity index 99% rename from litellm/llms/anthropic/pass_through/context_management/editors/compact.py rename to litellm/llms/anthropic/messages/context_management/editors/compact.py index 5eb2863a241..254d1c4c058 100644 --- a/litellm/llms/anthropic/pass_through/context_management/editors/compact.py +++ b/litellm/llms/anthropic/messages/context_management/editors/compact.py @@ -21,12 +21,12 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify -from litellm.llms.anthropic.pass_through.caller_model_checks import ( +from litellm.llms.anthropic.messages.caller_model_checks import ( caller_can_call_model, caller_within_model_budget, caller_within_model_rate_limit, ) -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.utils import ( ProxySubCallLoggingKwargs, proxy_general_settings, proxy_sub_call_metadata, @@ -269,7 +269,7 @@ def _count_effective_tokens( threshold check matches the downstream ``input_tokens`` metric. """ # Local import to avoid pulling the adapter at module load time. - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -426,7 +426,7 @@ def _build_summary_messages( system prompt); the conversation history is translated to OpenAI shape; the summarization prompt is appended as a final user turn. """ - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) diff --git a/litellm/llms/anthropic/pass_through/context_management/errors.py b/litellm/llms/anthropic/messages/context_management/errors.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/errors.py rename to litellm/llms/anthropic/messages/context_management/errors.py diff --git a/litellm/llms/anthropic/pass_through/context_management/placeholders.py b/litellm/llms/anthropic/messages/context_management/placeholders.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/placeholders.py rename to litellm/llms/anthropic/messages/context_management/placeholders.py diff --git a/litellm/llms/anthropic/pass_through/context_management/result.py b/litellm/llms/anthropic/messages/context_management/result.py similarity index 100% rename from litellm/llms/anthropic/pass_through/context_management/result.py rename to litellm/llms/anthropic/messages/context_management/result.py diff --git a/litellm/llms/anthropic/pass_through/convertible_blocks.py b/litellm/llms/anthropic/messages/convertible_blocks.py similarity index 99% rename from litellm/llms/anthropic/pass_through/convertible_blocks.py rename to litellm/llms/anthropic/messages/convertible_blocks.py index 9d651f8d721..09d662da543 100644 --- a/litellm/llms/anthropic/pass_through/convertible_blocks.py +++ b/litellm/llms/anthropic/messages/convertible_blocks.py @@ -12,7 +12,7 @@ from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id -from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages +from litellm.llms.anthropic.messages.native.utils import normalize_native_messages from litellm.types.utils import TokenCountMessagesFormat _CONVERTIBLE_BLOCK_TYPES: Final = MappingProxyType( diff --git a/litellm/llms/anthropic/pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py similarity index 98% rename from litellm/llms/anthropic/pass_through/messages/agentic_streaming_iterator.py rename to litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py index 922a6e21bae..25d4d7b3ce1 100644 --- a/litellm/llms/anthropic/pass_through/messages/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py @@ -336,7 +336,7 @@ class AgenticAnthropicStreamingIterator: await task async def aclose(self) -> None: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( aclose_if_supported, ) @@ -379,7 +379,7 @@ class AgenticAnthropicStreamingIterator: if hasattr(result, "__aiter__"): self._follow_up_iterator = result.__aiter__() elif isinstance(result, dict): - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( diff --git a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py rename to litellm/llms/anthropic/messages/native/fake_stream_iterator.py index 2b6161d98f6..39bdfbbc53a 100644 --- a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py @@ -12,7 +12,7 @@ import json from collections.abc import Mapping, Sequence from typing import Final, cast -from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta +from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/messages/native/handler.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/handler.py rename to litellm/llms/anthropic/messages/native/handler.py index a93e9106fb4..6f97a019a96 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -99,6 +99,7 @@ def _should_route_to_responses_api( requested_model: str | None = None, resolved_model: str | None = None, deployment_model_info: object = None, + stop_sequences: Sequence[str] | None = None, ) -> bool: """Return True when the request should use the Responses API path. @@ -107,6 +108,8 @@ def _should_route_to_responses_api( """ if litellm.use_chat_completions_url_for_anthropic_messages: return False + if stop_sequences: + return False if custom_llm_provider in _RESPONSES_API_PROVIDERS: return True if custom_llm_provider is None or requested_model is None or resolved_model is None: @@ -362,7 +365,7 @@ async def _try_websearch_short_circuit( ), ) if stream: - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) @@ -656,7 +659,7 @@ def anthropic_messages_handler( # reference the provider cannot resolve. Popped from kwargs so it never reaches the provider. skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False) if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list - from litellm.llms.anthropic.pass_through.messages.mcp_handler import ( + from litellm.llms.anthropic.messages.native.mcp_handler import ( anthropic_messages_with_mcp, ) @@ -690,7 +693,7 @@ def anthropic_messages_handler( if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if kwargs.get("compaction") is None and _should_route_to_responses_api( - custom_llm_provider, original_model, model, kwargs.get("model_info") + custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences ): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( max_tokens=max_tokens, diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/README.md b/litellm/llms/anthropic/messages/native/interceptors/README.md similarity index 100% rename from litellm/llms/anthropic/pass_through/messages/interceptors/README.md rename to litellm/llms/anthropic/messages/native/interceptors/README.md diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/__init__.py b/litellm/llms/anthropic/messages/native/interceptors/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/messages/interceptors/__init__.py rename to litellm/llms/anthropic/messages/native/interceptors/__init__.py diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/messages/native/interceptors/advisor.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py rename to litellm/llms/anthropic/messages/native/interceptors/advisor.py index 85d76de5402..f69d9820ef9 100644 --- a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/messages/native/interceptors/advisor.py @@ -68,7 +68,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): custom_llm_provider: str | None, **kwargs, ) -> AnthropicMessagesResponse | AsyncIterator: - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/base.py b/litellm/llms/anthropic/messages/native/interceptors/base.py similarity index 100% rename from litellm/llms/anthropic/pass_through/messages/interceptors/base.py rename to litellm/llms/anthropic/messages/native/interceptors/base.py diff --git a/litellm/llms/anthropic/pass_through/messages/mcp_handler.py b/litellm/llms/anthropic/messages/native/mcp_handler.py similarity index 95% rename from litellm/llms/anthropic/pass_through/messages/mcp_handler.py rename to litellm/llms/anthropic/messages/native/mcp_handler.py index e85733df779..8fe18d79f24 100644 --- a/litellm/llms/anthropic/pass_through/messages/mcp_handler.py +++ b/litellm/llms/anthropic/messages/native/mcp_handler.py @@ -76,9 +76,9 @@ async def anthropic_messages_with_mcp( upstream provider as a url it cannot reach. """ import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import request_safeguards_evaluator - from litellm.llms.anthropic.pass_through.messages.handler import proxy_classifies_tool_uses - from litellm.llms.anthropic.pass_through.safeguards import with_safeguard_results + from litellm.llms.anthropic.messages.adapters.handler import request_safeguards_evaluator + from litellm.llms.anthropic.messages.native.handler import proxy_classifies_tool_uses + from litellm.llms.anthropic.messages.safeguards import with_safeguard_results from litellm.mcp_client.tools import ( transform_mcp_tool_to_anthropic_tool, ) @@ -198,7 +198,7 @@ async def anthropic_messages_with_mcp( else response ) if stream: - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) diff --git a/litellm/llms/anthropic/pass_through/messages/mid_conversation_system.py b/litellm/llms/anthropic/messages/native/mid_conversation_system.py similarity index 100% rename from litellm/llms/anthropic/pass_through/messages/mid_conversation_system.py rename to litellm/llms/anthropic/messages/native/mid_conversation_system.py diff --git a/litellm/llms/anthropic/pass_through/messages/response_cache.py b/litellm/llms/anthropic/messages/native/response_cache.py similarity index 98% rename from litellm/llms/anthropic/pass_through/messages/response_cache.py rename to litellm/llms/anthropic/messages/native/response_cache.py index 81d01f1437e..d2425c3f484 100644 --- a/litellm/llms/anthropic/pass_through/messages/response_cache.py +++ b/litellm/llms/anthropic/messages/native/response_cache.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Final, cast import litellm from litellm._logging import verbose_logger from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( +from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, BaseAnthropicMessagesStreamingIterator, aclose_if_supported, diff --git a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/messages/native/streaming_iterator.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/streaming_iterator.py rename to litellm/llms/anthropic/messages/native/streaming_iterator.py index bfa309fad83..52b3f1042f8 100644 --- a/litellm/llms/anthropic/pass_through/messages/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/streaming_iterator.py @@ -18,13 +18,13 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( INCOMPLETE_STREAM_ERROR_MESSAGE, MESSAGE_DELTA_OPTIONAL_KEYS, anthropic_message_delta, ) -from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults -from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame +from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults +from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) @@ -33,7 +33,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointT from litellm.types.utils import GenericStreamingChunk, ModelResponseStream if TYPE_CHECKING: - from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging() diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/transformation.py rename to litellm/llms/anthropic/messages/native/transformation.py index 35db54147c5..cc53858f842 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -10,8 +10,8 @@ from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts -from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards +from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts +from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -736,7 +736,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): litellm_logging_obj: LiteLLMLoggingObj, ) -> AsyncIterator[bytes]: """Helper function to handle Anthropic streaming responses using the existing logging handlers""" - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) diff --git a/litellm/llms/anthropic/pass_through/messages/utils.py b/litellm/llms/anthropic/messages/native/utils.py similarity index 99% rename from litellm/llms/anthropic/pass_through/messages/utils.py rename to litellm/llms/anthropic/messages/native/utils.py index d91a5e2fb5c..29bbd8178ba 100644 --- a/litellm/llms/anthropic/pass_through/messages/utils.py +++ b/litellm/llms/anthropic/messages/native/utils.py @@ -11,7 +11,7 @@ from litellm.llms.anthropic.common_utils import ( sanitize_tool_use_ids_in_anthropic_messages, strip_empty_content_blocks_from_anthropic_messages, ) -from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled +from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.types.llms.anthropic import ( AnthropicMessagesRequestOptionalParams, AnthropicStopDetails, diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/__init__.py b/litellm/llms/anthropic/messages/responses_adapters/__init__.py similarity index 100% rename from litellm/llms/anthropic/pass_through/responses_adapters/__init__.py rename to litellm/llms/anthropic/messages/responses_adapters/__init__.py diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py similarity index 99% rename from litellm/llms/anthropic/pass_through/responses_adapters/handler.py rename to litellm/llms/anthropic/messages/responses_adapters/handler.py index b1347b0c79e..99a07a1e9bf 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -201,7 +201,7 @@ def _build_responses_kwargs( if isinstance(reasoning, dict): effort: Final[object] = reasoning.get("effort") if isinstance(effort, str): - from litellm.llms.anthropic.pass_through.utils import ( + from litellm.llms.anthropic.messages.utils import ( normalize_reasoning_effort_value, ) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py similarity index 98% rename from litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py rename to litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py index e628a30fdd6..614a7c96848 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py @@ -14,13 +14,13 @@ from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( INCOMPLETE_STREAM_ERROR_MESSAGE, refusal_stop_details, responses_output_refusal_text, ) -from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults -from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame +from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults +from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.responses.streaming_iterator import ( partial_responses_incomplete_event_on_disconnect, stream_error_status_and_message, @@ -35,7 +35,7 @@ from .transformation import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject - from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator + from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.types.llms.openai import ResponseIncompleteEvent diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py similarity index 99% rename from litellm/llms/anthropic/pass_through/responses_adapters/transformation.py rename to litellm/llms/anthropic/messages/responses_adapters/transformation.py index 48fc987a95a..1dd07c1739b 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -20,12 +20,12 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, ) -from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_parts +from litellm.llms.anthropic.messages.native.utils import ( refusal_stop_details, responses_output_refusal_text, ) -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.utils import ( is_reasoning_auto_summary_enabled, json_copy_or_original, prompt_cache_key_from_user_id, @@ -71,7 +71,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if raw_usage is None: return AnthropicUsage(input_tokens=0, output_tokens=0) - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) from litellm.responses.utils import ResponseAPILoggingUtils diff --git a/litellm/llms/anthropic/pass_through/safeguards.py b/litellm/llms/anthropic/messages/safeguards.py similarity index 99% rename from litellm/llms/anthropic/pass_through/safeguards.py rename to litellm/llms/anthropic/messages/safeguards.py index eededde7e92..22627a454b3 100644 --- a/litellm/llms/anthropic/pass_through/safeguards.py +++ b/litellm/llms/anthropic/messages/safeguards.py @@ -11,8 +11,8 @@ from typing_extensions import NotRequired, ReadOnly, Unpack, assert_never import litellm from litellm._logging import verbose_logger -from litellm.llms.anthropic.pass_through.caller_model_checks import CallerModelVerdict, caller_model_verdict -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.caller_model_checks import CallerModelVerdict, caller_model_verdict +from litellm.llms.anthropic.messages.utils import ( ProxySubCallLoggingKwargs, proxy_general_settings, proxy_sub_call_metadata, diff --git a/litellm/llms/anthropic/pass_through/stream_assembly.py b/litellm/llms/anthropic/messages/stream_assembly.py similarity index 99% rename from litellm/llms/anthropic/pass_through/stream_assembly.py rename to litellm/llms/anthropic/messages/stream_assembly.py index 90cb9f4c335..90bdad8f4bc 100644 --- a/litellm/llms/anthropic/pass_through/stream_assembly.py +++ b/litellm/llms/anthropic/messages/stream_assembly.py @@ -12,7 +12,7 @@ from litellm.llms.anthropic.chat.handler import ( ModelResponseIterator as AnthropicModelResponseIterator, ) from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.anthropic.pass_through.messages.utils import MESSAGE_DELTA_OPTIONAL_KEYS +from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS from litellm.types.utils import ( Choices, Message, diff --git a/litellm/llms/anthropic/pass_through/utils.py b/litellm/llms/anthropic/messages/utils.py similarity index 100% rename from litellm/llms/anthropic/pass_through/utils.py rename to litellm/llms/anthropic/messages/utils.py diff --git a/litellm/llms/anthropic/pass_through/__init__.py b/litellm/llms/anthropic/pass_through/__init__.py new file mode 100644 index 00000000000..c29255c09c2 --- /dev/null +++ b/litellm/llms/anthropic/pass_through/__init__.py @@ -0,0 +1,7 @@ +from litellm.litellm_core_utils.renamed_package_shim import alias_renamed_package + +__getattr__ = alias_renamed_package( + __name__, + "litellm.llms.anthropic.messages", + nested_renames=((f"{__name__}.messages", "litellm.llms.anthropic.messages.native"),), +) diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 4c363dba860..7a4c707860d 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -24,11 +24,11 @@ from litellm.llms.anthropic.common_utils import ( ) from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) -from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils, prepare_native_messages +from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages from litellm.router_utils.baseline_request import ( BASELINE_PARAMETERS, capture_baseline_parameters, diff --git a/litellm/llms/azure/decisions/transformation.py b/litellm/llms/azure/decisions/transformation.py new file mode 100644 index 00000000000..3fb40e39702 --- /dev/null +++ b/litellm/llms/azure/decisions/transformation.py @@ -0,0 +1,77 @@ +import asyncio +from collections.abc import Mapping +from typing import Final + +from pydantic import ValidationError + +import litellm +from litellm.llms.azure.common_utils import ( + BaseAzureLLM, + cached_azure_ad_token_refresh_provider, +) +from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + + +class AzureDecisionsConfig(OpenAIDecisionsConfig): + """Azure OpenAI Decisions, with the Azure deployment name in the request body.""" + + api_key_required = False + + def resolve_api_base(self, api_base: str | None) -> str | None: + return BaseAzureLLM.resolve_azure_api_base(api_base) + + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key + + def resolve_credential( + self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object] + ) -> str | None: + return api_key + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> Mapping[str, str]: + try: + params: Final = GenericLiteLLMParams.model_validate({**litellm_params, "api_key": api_key}) + except ValidationError as error: + raise litellm.BadRequestError( + message="Invalid Azure Decisions credential parameters", + model=model, + llm_provider="azure", + ) from error + clean_headers: Final = super().validate_environment( + {name: value for name, value in headers.items() if name.lower() != "api-key"}, + model, + None, + litellm_params, + ) + azure_headers: Final = BaseAzureLLM.base_validate_azure_environment( + headers=dict(clean_headers), litellm_params=params + ) + if "api-key" in azure_headers: + return azure_headers + if "Authorization" in azure_headers: + return azure_headers + scope: Final = ( + params.azure_scope or get_secret_str("AZURE_SCOPE") or "https://cognitiveservices.azure.com/.default" + ) + azure_headers["Authorization"] = f"Bearer {cached_azure_ad_token_refresh_provider(scope)()}" + return azure_headers + + async def avalidate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> Mapping[str, str]: + return await asyncio.to_thread(self.validate_environment, headers, model, api_key, litellm_params) + + def get_complete_url(self, api_base: str, model: str) -> str: + return BaseAzureLLM.get_base_azure_url(api_base, None, route="/openai/v1/decisions") diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 2aacc5771d0..6b2da96de23 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -5,7 +5,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig from collections.abc import Sequence from typing import Any, Final -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure.common_utils import BaseAzureLLM diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 14c6a2d2a66..b3df4654a6b 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -351,9 +351,10 @@ def _ir_score_answer(question: DecisionsIRScoreQuestion, answer: ScoreAnswer) -> DecisionsIRScoreProbability( value=index, label=answer.legend.get(str(index), level.label), - probability=answer.probabilities.get(str(index), 0.0), + probability=answer.probabilities[str(index)], ) for index, level in enumerate(question.levels) + if str(index) in answer.probabilities ), extra=MappingProxyType(answer.model_extra or {}), ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 4d7ec663d09..0bc07ac6b22 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1955,7 +1955,7 @@ class AmazonConverseConfig(BaseConfig): anthropic_beta_list: list, ) -> None: """Keep only compact_20260112 edits for Bedrock; add beta header or drop field.""" - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_EDIT_TYPE, ) from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES diff --git a/litellm/llms/bedrock/claude_platform/messages_transformation.py b/litellm/llms/bedrock/claude_platform/messages_transformation.py index 57ebaa0ca71..06308cde124 100644 --- a/litellm/llms/bedrock/claude_platform/messages_transformation.py +++ b/litellm/llms/bedrock/claude_platform/messages_transformation.py @@ -1,7 +1,7 @@ from typing import Final import litellm -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) diff --git a/litellm/llms/bedrock/count_tokens/kimi_k3.py b/litellm/llms/bedrock/count_tokens/kimi_k3.py index 4f8b4624cd8..5eb46944f9a 100644 --- a/litellm/llms/bedrock/count_tokens/kimi_k3.py +++ b/litellm/llms/bedrock/count_tokens/kimi_k3.py @@ -32,7 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( chat_completion_tools_for, is_flat_responses_tool, ) -from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam KIMI_K3_REPO: Final = "moonshotai/Kimi-K3" diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py index 805652630ed..008b977f485 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py @@ -24,7 +24,7 @@ from litellm.llms.anthropic.common_utils import ( carried_thinking_keys, validated_object_mapping, ) -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -878,7 +878,7 @@ class AmazonAnthropicClaudeMessagesConfig( merge them from ``message_start`` so logging/cost sees a consistent usage object (fixes negative input costs: LIT-2411). """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) diff --git a/litellm/llms/bedrock/messages/mantle_transformation.py b/litellm/llms/bedrock/messages/mantle_transformation.py index 7b9dde42c4a..89f4cc4dd8b 100644 --- a/litellm/llms/bedrock/messages/mantle_transformation.py +++ b/litellm/llms/bedrock/messages/mantle_transformation.py @@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import TypeAdapter -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( DEFAULT_ANTHROPIC_API_VERSION, AnthropicMessagesConfig, ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 7b808cf2be2..91ac5a0ca9a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -238,7 +238,7 @@ if TYPE_CHECKING: from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig @@ -2650,7 +2650,7 @@ class BaseLLMHTTPHandler: initial_response: AsyncIterator | AnthropicMessagesResponse if stream: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, anthropic_messages_stream_hidden_params, ) @@ -2674,7 +2674,7 @@ class BaseLLMHTTPHandler: hidden_params=stream_hidden_params, ) - from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) @@ -6237,7 +6237,7 @@ class BaseLLMHTTPHandler: from typing import cast from litellm._logging import verbose_logger - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( diff --git a/litellm/llms/databricks/agent/transformation.py b/litellm/llms/databricks/agent/transformation.py index fb0fdd22132..7747837a68b 100644 --- a/litellm/llms/databricks/agent/transformation.py +++ b/litellm/llms/databricks/agent/transformation.py @@ -5,16 +5,14 @@ Model Serving, unified: POST {workspace}/serving-endpoints/responses with the en Databricks Apps: POST {app}/responses """ -import hashlib from collections.abc import AsyncIterator, Iterator, Mapping, Sequence -from typing import TYPE_CHECKING, Final, Protocol +from typing import TYPE_CHECKING, Final from urllib.parse import quote, urlsplit import httpx from pydantic import BaseModel, ConfigDict, StrictStr, TypeAdapter, ValidationError import litellm -from litellm.caching.in_memory_cache import InMemoryCache from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) @@ -41,20 +39,6 @@ _ENDPOINT_URL_SUFFIXES: Final = (_INVOCATIONS_SUFFIX, _APP_RESPONSES_SUFFIX) _SERVING_ENDPOINTS_SEGMENT: Final = "/serving-endpoints/" _UNIFIED_RESPONSES_SUFFIX: Final = "/serving-endpoints/responses" _APP_HOST_SUFFIX: Final = ".databricksapps.com" -_TOKEN_EXPIRY_BUFFER_SECONDS: Final = 60 - - -class _TokenCache(Protocol): - def get_cache(self, key: str) -> object: ... - - def set_cache(self, key: str, value: str, *, ttl: int) -> None: ... - - -def _new_token_cache() -> _TokenCache: - return InMemoryCache(max_size_in_memory=64) - - -_ENV_M2M_TOKENS: Final = _new_token_cache() _INPUT_ROLES: Final = frozenset({"system", "developer", "user", "assistant"}) _BODY_PARAMS: Final = ( "max_output_tokens", @@ -220,20 +204,7 @@ def _env_oauth_m2m_token(api_base: str | None) -> str | None: "set DATABRICKS_HOST to the workspace URL (a Databricks App URL is not a workspace)" ), ) - return _cached_m2m_token(workspace, client_id, client_secret) - - -def _cached_m2m_token(workspace: str, client_id: str, client_secret: str) -> str: - secret_digest: Final = hashlib.sha256(client_secret.encode()).hexdigest()[:16] - cache_key: Final = f"{workspace}|{client_id}|{secret_digest}" - cached: Final[object] = _ENV_M2M_TOKENS.get_cache(cache_key) - if isinstance(cached, str): - return cached - token: Final = DatabricksBase().get_oauth_m2m_token_and_lifetime(workspace, client_id, client_secret) - ttl: Final = token.expires_in - _TOKEN_EXPIRY_BUFFER_SECONDS - if ttl > 0: - _ENV_M2M_TOKENS.set_cache(cache_key, token.access_token, ttl=ttl) - return token.access_token + return DatabricksBase().get_cached_oauth_m2m_token(workspace, client_id, client_secret) def _resolved_env_reference(value: str | None) -> str | None: diff --git a/litellm/llms/databricks/common_utils.py b/litellm/llms/databricks/common_utils.py index cd9432ac36f..b1562bef6b8 100644 --- a/litellm/llms/databricks/common_utils.py +++ b/litellm/llms/databricks/common_utils.py @@ -10,21 +10,33 @@ Authentication priority: 3. Databricks SDK automatic auth - Fallback (uses unified auth) """ +import hashlib import os import re from collections.abc import Mapping from types import MappingProxyType -from typing import Final, Literal +from typing import Final, Literal, Protocol from urllib.parse import urlsplit, urlunsplit from pydantic import BaseModel, ConfigDict import litellm +from litellm.caching.in_memory_cache import InMemoryCache from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str _DEFAULT_M2M_TOKEN_LIFETIME_SECONDS: Final = 3600 +_TOKEN_EXPIRY_BUFFER_SECONDS: Final = 60 + + +class _TokenCache(Protocol): + def get_cache(self, key: str) -> object: ... + + def set_cache(self, key: str, value: str, *, ttl: int) -> None: ... + + +_OAUTH_TOKENS: Final[_TokenCache] = InMemoryCache(max_size_in_memory=64) class DatabricksException(BaseLLMException): @@ -293,6 +305,26 @@ class DatabricksBase: ) -> str: return self.get_oauth_m2m_token_and_lifetime(api_base, client_id, client_secret).access_token + def get_cached_oauth_m2m_token( + self, + api_base: str, + client_id: str, + client_secret: str, + *, + token_cache: _TokenCache = _OAUTH_TOKENS, + ) -> str: + workspace: Final = self.workspace_root(api_base) + secret_digest: Final = hashlib.sha256(client_secret.encode()).hexdigest() + cache_key: Final = f"{workspace}|{client_id}|{secret_digest}" + cached: Final = token_cache.get_cache(cache_key) + if isinstance(cached, str): + return cached + token: Final = self.get_oauth_m2m_token_and_lifetime(workspace, client_id, client_secret) + ttl: Final = token.expires_in - _TOKEN_EXPIRY_BUFFER_SECONDS + if ttl > 0: + token_cache.set_cache(cache_key, token.access_token, ttl=ttl) + return token.access_token + def get_oauth_m2m_token_and_lifetime( self, api_base: str, diff --git a/litellm/llms/databricks/decisions/transformation.py b/litellm/llms/databricks/decisions/transformation.py index d1698fdf5e2..2b0524da2f9 100644 --- a/litellm/llms/databricks/decisions/transformation.py +++ b/litellm/llms/databricks/decisions/transformation.py @@ -8,6 +8,8 @@ from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.databricks.common_utils import DatabricksBase +from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import DecisionsIRRequest, UnsupportedDecisionsRequest DATABRICKS_AI_DECIDE_MODEL: Final = "ai_decide" @@ -74,6 +76,17 @@ class DatabricksDecisionsConfig(BaseDecisionsConfig): api_key_env = ("DATABRICKS_API_KEY", "DATABRICKS_TOKEN") api_base_env = ("DATABRICKS_API_BASE",) + def resolve_credential( + self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object] + ) -> str | None: + if api_key: + return api_key + client_id: Final = get_secret_str("DATABRICKS_CLIENT_ID") + client_secret: Final = get_secret_str("DATABRICKS_CLIENT_SECRET") + if not client_id or not client_secret: + return None + return DatabricksBase().get_cached_oauth_m2m_token(api_base, client_id, client_secret) + def missing_api_base_message(self, custom_llm_provider: str) -> str: return ( f"api_base is required for Decisions provider '{custom_llm_provider}': set DATABRICKS_API_BASE to " diff --git a/litellm/llms/databricks/messages/transformation.py b/litellm/llms/databricks/messages/transformation.py index f5ccbd92fb4..e22ad85b5f9 100644 --- a/litellm/llms/databricks/messages/transformation.py +++ b/litellm/llms/databricks/messages/transformation.py @@ -8,7 +8,7 @@ from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.databricks.common_utils import ( diff --git a/litellm/llms/deepseek/messages/transformation.py b/litellm/llms/deepseek/messages/transformation.py index 3b2dd5af260..3b2076b0085 100644 --- a/litellm/llms/deepseek/messages/transformation.py +++ b/litellm/llms/deepseek/messages/transformation.py @@ -9,7 +9,7 @@ from pydantic import TypeAdapter import litellm from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.secret_managers.main import get_secret_str diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index 613b7e42100..bc2b68cd02b 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final, cast # noqa: TID251 # narrows untyped request dicts from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/litellm/llms/lm_studio/decisions/transformation.py b/litellm/llms/lm_studio/decisions/transformation.py new file mode 100644 index 00000000000..84e5ff14443 --- /dev/null +++ b/litellm/llms/lm_studio/decisions/transformation.py @@ -0,0 +1,16 @@ +from typing import Final + +from litellm.constants import PLACEHOLDER_API_KEY +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class LMStudioDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("LM_STUDIO_API_KEY",) + api_base_env = ("LM_STUDIO_API_BASE",) + api_key_required = False + supports_images = True + inlines_remote_images = True + + def resolve_api_key(self, api_key: str | None) -> str | None: + resolved: Final = super().resolve_api_key(api_key) + return None if resolved == PLACEHOLDER_API_KEY else resolved diff --git a/litellm/llms/minimax/messages/transformation.py b/litellm/llms/minimax/messages/transformation.py index f7a263a467c..f546d2d3a16 100644 --- a/litellm/llms/minimax/messages/transformation.py +++ b/litellm/llms/minimax/messages/transformation.py @@ -4,7 +4,7 @@ MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's import litellm from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.secret_managers.main import get_secret_str diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 25d403a0dff..4518fe6a77e 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -270,10 +270,9 @@ def _openai_choice_answer(question: DecisionsIRChoiceQuestion, answer: Decisions name=question.name, choice=answer.choice, probabilities=tuple( - OpenAIChoiceProbability( - value=option.value, probability=probabilities.get(systemone_choice_key(option.value), 0.0) - ) + OpenAIChoiceProbability(value=option.value, probability=probabilities[systemone_choice_key(option.value)]) for option in question.choices + if systemone_choice_key(option.value) in probabilities ), confidence=answer.confidence, ) @@ -285,10 +284,9 @@ def _openai_score_answer(question: DecisionsIRScoreQuestion, answer: DecisionsIR name=question.name, score=answer.score, probabilities=tuple( - OpenAIScoreProbability( - value=index, label=decisions_text(level.label), probability=probabilities.get(index, 0.0) - ) + OpenAIScoreProbability(value=index, label=decisions_text(level.label), probability=probabilities[index]) for index, level in enumerate(question.levels) + if index in probabilities ), confidence=answer.confidence, ) diff --git a/litellm/llms/openai/responses/count_tokens/token_counter.py b/litellm/llms/openai/responses/count_tokens/token_counter.py index deee502c9f2..6ae61e15953 100644 --- a/litellm/llms/openai/responses/count_tokens/token_counter.py +++ b/litellm/llms/openai/responses/count_tokens/token_counter.py @@ -15,8 +15,8 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.tool_fields import DropPolicy -from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.base_llm.base_utils import BaseTokenCounter diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 0f885146e36..b4e152ff406 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -15,9 +15,9 @@ from typing_extensions import ReadOnly, TypedDict from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error -from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.convertible_blocks import malformed_tool_result_text_error +from litellm.llms.anthropic.messages.native.utils import normalize_native_messages +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.responses.utils import ResponsesAPIRequestUtils diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 0ab55d653e0..8daae2a8e6a 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final import litellm from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig diff --git a/litellm/llms/opencode/messages/transformation.py b/litellm/llms/opencode/messages/transformation.py index b08a6df47aa..16953a33329 100644 --- a/litellm/llms/opencode/messages/transformation.py +++ b/litellm/llms/opencode/messages/transformation.py @@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence from pydantic import TypeAdapter from litellm.litellm_core_utils.url_utils import ensure_api_route -from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig +from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig from litellm.llms.opencode.common_utils import go_api_base, go_headers diff --git a/litellm/llms/tencent/messages/transformation.py b/litellm/llms/tencent/messages/transformation.py index 30c3ecaf3a9..cfc9e9ed6aa 100644 --- a/litellm/llms/tencent/messages/transformation.py +++ b/litellm/llms/tencent/messages/transformation.py @@ -7,7 +7,7 @@ alongside its standard OpenAI-compatible chat completions endpoint. import litellm from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.secret_managers.main import get_secret_str diff --git a/litellm/llms/vertex_ai/count_tokens/context.py b/litellm/llms/vertex_ai/count_tokens/context.py index 2579e857bd9..fc7d6545624 100644 --- a/litellm/llms/vertex_ai/count_tokens/context.py +++ b/litellm/llms/vertex_ai/count_tokens/context.py @@ -134,7 +134,7 @@ def vertex_gemini_count_context( def unconvertible_messages_error( messages: Sequence[Mapping[str, object]] | None, messages_format: TokenCountMessagesFormat ) -> VertexGeminiCountContextError | None: - from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error + from litellm.llms.anthropic.messages.convertible_blocks import unconvertible_blocks_error error_message: Final = unconvertible_blocks_error(messages, messages_format) return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) @@ -145,7 +145,7 @@ def malformed_tool_result_text_count_error( messages_format: TokenCountMessagesFormat, deployment: Mapping[str, object] | None, ) -> VertexGeminiCountContextError | None: - from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_count_error + from litellm.llms.anthropic.messages.convertible_blocks import malformed_tool_result_count_error error_message: Final = malformed_tool_result_count_error(messages, messages_format, deployment) return None if error_message is None else VertexGeminiCountContextError(error_message=error_message) @@ -235,7 +235,7 @@ def _view_without_edits( system: object | None, typed_system: str | Sequence[Mapping[str, object]] | None, ) -> InferenceRequestView: - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( apply_client_compaction_block_history, ) @@ -253,9 +253,9 @@ def _view_without_edits( async def _anthropic_inference_view( messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits ) -> InferenceRequestView | VertexGeminiCountContextError: - from litellm.llms.anthropic.pass_through.adapters.handler import prepare_context_managed_request - from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError - from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages + from litellm.llms.anthropic.messages.adapters.handler import prepare_context_managed_request + from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError + from litellm.llms.anthropic.messages.native.utils import normalize_native_messages typed_system: Final = _typed_system(system) if isinstance(typed_system, VertexGeminiCountContextError): @@ -308,7 +308,7 @@ def _chat_messages( model: str, custom_llm_provider: Literal["vertex_ai", "gemini"], ) -> Sequence[AllMessageValues]: - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter match messages_format: case "anthropic": diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py new file mode 100644 index 00000000000..9756aa4ec5a --- /dev/null +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py @@ -0,0 +1,3 @@ +from litellm.litellm_core_utils.renamed_package_shim import alias_renamed_package + +__getattr__ = alias_renamed_package(__name__, "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages") diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py similarity index 61% rename from litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py rename to litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index cfd5b5d3980..6d8eb228786 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -1,8 +1,8 @@ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Final from litellm.llms.anthropic.common_utils import AnthropicModelInfo -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, ) @@ -26,34 +26,92 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert def should_strip_billing_metadata(self) -> bool: return True + @staticmethod + def _vertex_auth_parameters( + litellm_params: dict[str, object], # mutable-ok: delegates to the existing Vertex parameter readers + ) -> tuple[str | None, str | None, str | None]: + return ( + VertexBase.safe_get_vertex_ai_project(litellm_params), + VertexBase.safe_get_vertex_ai_location(litellm_params), + VertexBase.safe_get_vertex_ai_credentials(litellm_params), + ) + def validate_anthropic_messages_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: Sequence[object], - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> tuple[dict, str | None]: + ) -> tuple[dict[str, str], str | None]: """ OPTIONAL Validate the environment for the request """ - # Work on a local copy — router shallow-copies litellm_params so the caller's - # headers dict may be the shared deployment extra_headers object. - headers = dict(headers) - vertex_ai_project: Final = VertexBase.safe_get_vertex_ai_project(litellm_params) - vertex_ai_location: Final = VertexBase.safe_get_vertex_ai_location(litellm_params) - - vertex_credentials: Final = VertexBase.safe_get_vertex_ai_credentials(litellm_params) + vertex_ai_project, vertex_ai_location, vertex_credentials = self._vertex_auth_parameters(litellm_params) access_token, project_id = self._ensure_access_token( credentials=vertex_credentials, project_id=vertex_ai_project, custom_llm_provider="vertex_ai", ) - headers["Authorization"] = f"Bearer {access_token}" + return self._finalize_vertex_messages_environment( + headers, + model, + messages, + optional_params, + api_base, + access_token, + project_id, + vertex_ai_project, + vertex_ai_location, + ) + + async def avalidate_anthropic_messages_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors the provider validation contract + model: str, + messages: Sequence[object], + optional_params: dict[str, object], # mutable-ok: mirrors the provider validation contract + litellm_params: dict[str, object], # mutable-ok: mirrors the provider validation contract + api_key: str | None = None, + api_base: str | None = None, + ) -> tuple[dict[str, str], str | None]: # mutable-ok: the caller consumes request headers + vertex_ai_project, vertex_ai_location, vertex_credentials = self._vertex_auth_parameters(litellm_params) + access_token, project_id = await self._ensure_access_token_async( + credentials=vertex_credentials, + project_id=vertex_ai_project, + custom_llm_provider="vertex_ai", + ) + return self._finalize_vertex_messages_environment( + headers, + model, + messages, + optional_params, + api_base, + access_token, + project_id, + vertex_ai_project, + vertex_ai_location, + ) + + def _finalize_vertex_messages_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[object], + optional_params: dict, # mutable-ok: preserves the existing provider optional-parameter contract + api_base: str | None, + access_token: str, + project_id: str, + vertex_ai_project: str | None, + vertex_ai_location: str | None, + ) -> tuple[dict[str, str], str | None]: # mutable-ok: the caller consumes request headers + # Router shallow-copies litellm_params, so headers may be shared between requests. + request_headers: Final = dict(headers) + request_headers["Authorization"] = f"Bearer {access_token}" api_base = self.get_complete_vertex_url( custom_api_base=api_base, @@ -65,14 +123,14 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert model=model, ) - headers["content-type"] = "application/json" + request_headers["content-type"] = "application/json" # Add beta headers for Vertex AI tools: Final = optional_params.get("tools", []) beta_values: Final[set[str]] = set() # Get existing beta headers if any - existing_beta: Final = headers.get("anthropic-beta") + existing_beta: Final = request_headers.get("anthropic-beta") if existing_beta: beta_values.update(b.strip() for b in existing_beta.split(",")) @@ -117,9 +175,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) if beta_values: - headers["anthropic-beta"] = ",".join(beta_values) + request_headers["anthropic-beta"] = ",".join(beta_values) - return headers, api_base + return request_headers, api_base def get_complete_url( self, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py index db27427a5be..00c28a728e0 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py @@ -2,7 +2,7 @@ Shared sanitization for ``output_config`` / ``output_format`` on Vertex AI Claude. Lives in its own module so both the chat-completion transformation (``transformation.py``) and the Messages pass-through transformation -(``experimental_pass_through/transformation.py``) can import it without +(``messages/transformation.py``) can import it without forming a cycle through the parent module's heavier imports. CodeQL flagged the ``..transformation`` import path as a potential cyclic diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 2e9d6a77a62..0e2fc2e3dcb 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -5,7 +5,7 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider -from litellm.llms.anthropic.pass_through.messages import handler as main +from litellm.llms.anthropic.messages.native import handler as main from litellm.rust_bridge import detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target from litellm.rust_bridge.dispatch import PublicDispatch diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index b12676e08f3..3d2f89be15f 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -19,7 +19,7 @@ from litellm.anthropic_interface.exceptions import ( ) from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.tool_fields import first_validation_error -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, ) from litellm.llms.base_llm.guardrail_translation.utils import ( diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 61cf4d696a1..8e681c615ec 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -4687,6 +4687,7 @@ class ProxyBaseLLMRequestProcessing: total_tokens=total_tokens, completion_tokens_details=usage.get("completion_tokens_details"), prompt_tokens_details=usage.get("prompt_tokens_details"), + server_tool_use=usage.get("server_tool_use"), ) @staticmethod diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 2baeabdce7b..3f19f5cb386 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -13,15 +13,15 @@ from types import MappingProxyType from typing import Final from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details -from litellm.llms.anthropic.pass_through.stream_assembly import anthropic_message_delta_fields -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details +from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields +from litellm.llms.anthropic.messages.stream_assembly import ( assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export ) -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.stream_assembly import ( is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export ) -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.stream_assembly import ( is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export ) from litellm.types.utils import Choices, ModelResponse @@ -50,7 +50,7 @@ def anthropic_sse_error_frames(message: str) -> tuple[bytes, ...]: ) -from litellm.llms.anthropic.pass_through.stream_assembly import ( # noqa: E402 # re-export after the definitions above +from litellm.llms.anthropic.messages.stream_assembly import ( # noqa: E402 # re-export after the definitions above is_sse_error_stream as is_sse_error_stream, # noqa: PLC0414 # public re-export ) @@ -111,10 +111,10 @@ def anthropic_sse_chunks_from_response( the input the classifier judged. Their free-text ``explanation`` was written about the text the guardrail rewrote, so it is dropped unless the caller rewrites no text (``keep_explanations``). """ - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py index f2f3ec201f5..589b865618b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py +++ b/litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py @@ -559,7 +559,7 @@ def _v3_answer(request_data: Mapping[str, object], model: str | None) -> Mapping return _v3_text_completion_as_chat(response) if not isinstance(response, ModelResponse) or not _v3_anthropic_messages_route(request_data): return _jsonable_dict(response) - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 15f70073c7a..c7aa530434a 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -21,7 +21,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, ) from litellm.llms.anthropic import get_anthropic_config -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.stream_assembly import ( build_complete_streaming_response, build_complete_streaming_response_legacy, build_usage_only_response_from_chunks, diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index b6a067b7666..c94dc16a8f5 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -311,7 +311,7 @@ class PassThroughStreamingHandler: @staticmethod def _error_frame_closed_at(endpoint_type: EndpointType, raw_bytes: Sequence[bytes]) -> tuple[str, str, int] | None: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( closing_superseded_attempt, parse_anthropic_error_event, ) @@ -342,7 +342,7 @@ class PassThroughStreamingHandler: - Vertex AI - OpenAI """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( is_message_stop_chunk, # pyright: ignore[reportPrivateUsage] # both native stream paths share terminal-event detection is_provider_error_chunk, # pyright: ignore[reportPrivateUsage] # provider errors must not become cache evidence ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 35f6acb0fbd..bc79becf7bb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -374,7 +374,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) -from litellm.llms.anthropic.pass_through.convertible_blocks import ( +from litellm.llms.anthropic.messages.convertible_blocks import ( deployment_drops_params, drop_non_string_tool_result_texts, malformed_tool_result_count_error, @@ -15171,7 +15171,7 @@ def _router_default_litellm_params(router: Router | None) -> Mapping[str, object def _count_context_management( context_management: JsonValue, model: str, request_litellm_params: Mapping[str, JsonValue] ) -> CountContextManagement | None: - from litellm.llms.anthropic.pass_through.adapters.handler import context_management_explicitly_dropped + from litellm.llms.anthropic.messages.adapters.handler import context_management_explicitly_dropped if context_management is None or llm_router is None: return None diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index b264af780a9..0efd4e4c619 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -234,6 +234,9 @@ class DefaultTeamSettingsResponse(SettingsResponse): class UIThemeSettingsResponse(SettingsResponse): """Response model for UI theme settings""" + logo_url_via_proxy: bool = False + logo_url_dark_via_proxy: bool = False + _TEAM_ADMIN_FIELD_ENUM: Final = tuple(sorted(SUPPORTED_TEAM_ADMIN_PERMISSIONS)) @@ -1377,7 +1380,7 @@ async def update_sso_settings( tags=["UI Theme Settings"], response_model=UIThemeSettingsResponse, ) -async def get_ui_theme_settings(): +async def get_ui_theme_settings() -> UIThemeSettingsResponse: """ Get UI theme configuration from the litellm_settings. Returns current logo settings for UI customization. @@ -1390,18 +1393,23 @@ async def get_ui_theme_settings(): # Load existing config config: Final = await proxy_config.get_config() - result: Final = await _get_settings_with_schema( - settings_key="ui_theme_config", - settings_class=UIThemeConfig, - config=config, + result: Final = _SettingsWithSchema.model_validate( + await _get_settings_with_schema( + settings_key="ui_theme_config", + settings_class=UIThemeConfig, + config=config, + ) ) - - stored_values: Final = result.get("values", {}) - result["values"] = { - **stored_values, - **{field: _resolve_ui_theme_field(stored_values, field) for field in _UI_THEME_FIELD_ENV_VARS}, + values: Final = { + **result.values, + **{field: _resolve_ui_theme_field(result.values, field) for field in _UI_THEME_FIELD_ENV_VARS}, } - return result + return UIThemeSettingsResponse( + values=values, + field_schema=result.field_schema, + logo_url_via_proxy=values["logo_url"] is None and bool(os.getenv("UI_LOGO_PATH", "").strip()), + logo_url_dark_via_proxy=values["logo_url_dark"] is None and bool(os.getenv("UI_LOGO_PATH_DARK", "").strip()), + ) def _validate_public_image_url(value: str | None, field_name: str) -> None: diff --git a/litellm/router.py b/litellm/router.py index 8d95e3e91d4..0d4045e6ca8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -737,7 +737,7 @@ def _anthropic_stream_forwards_ping_live(chunk: object, has_generated_content: b """A `ping` keepalive reaches the client live whenever the stream has not committed: it carries no lifecycle, so it cannot create overlapping lifecycles on the wire, and it keeps the connection alive while lifecycle frames sit buffered for a possible fallback during a long thinking pass.""" - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import is_anthropic_ping_chunk + from litellm.llms.anthropic.messages.native.streaming_iterator import is_anthropic_ping_chunk return not has_generated_content and is_anthropic_ping_chunk(chunk) @@ -753,7 +753,7 @@ def _without_line_breaks(value: object) -> str: def _anthropic_stream_error_is_gateway_verdict(chunk: object) -> bool: """AgenticAnthropicStreamingIterator's own retrieval-failure frame is the gateway's verdict, not a provider failure: another deployment would rerun the same failed hook, so it reaches the client instead of falling back.""" - from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( is_server_fulfilled_tool_leak_error, ) @@ -917,7 +917,7 @@ def _stream_commits_now(chunk: object, has_generated_content: bool, buffered_chu pre-content buffer cap was hit) rather than keep buffering lifecycle frames toward a possible fallback. """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( is_anthropic_content_delta_chunk, ) @@ -6307,7 +6307,7 @@ class Router: response=response, kwargs=kwargs, ): - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( safeguard_refusal_error, ) @@ -6424,16 +6424,16 @@ class Router: including pings and provider error frames, is forwarded live and verbatim instead. """ - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( aclose_if_supported, aclose_superseded_attempt, parse_anthropic_error_event, parse_anthropic_refusal_stop_details, ) - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( safeguard_refusal_error, ) @@ -6797,7 +6797,7 @@ class Router: A retry stamps the retry headers async_function_with_retries would have for a pre-stream retry; a later hop the recovered stream makes replaces them with its own, the way a fallback's do. """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( aclose_if_supported, anthropic_messages_response_as_sse_events, ) @@ -9931,7 +9931,7 @@ class Router: when a content-policy fallback is configured; a plain refusal without stop_details, or any response with nothing configured, is returned to the client unchanged. """ - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( get_safeguard_refusal_stop_details, ) @@ -13136,6 +13136,7 @@ class Router: healthy_deployments=remaining, request_kwargs=tag_request, metadata_variable_name=metadata_variable_name, + request_deployments=request_deployments, ) except ValueError: # tag routing sends this request to none of the rest break @@ -14147,7 +14148,7 @@ class Router: `tools` (Chat Completions, Responses and Anthropic Messages shapes) and the Anthropic Messages top-level `system` block. """ - from litellm.llms.anthropic.pass_through.messages.utils import ( + from litellm.llms.anthropic.messages.native.utils import ( anthropic_system_to_openai_message, ) @@ -15100,6 +15101,7 @@ class Router: request_kwargs=request_kwargs, healthy_deployments=healthy_deployments, metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs), + request_deployments=request_deployments, ) # narrow to whatever `self.routing_plugins` left in candidate_models diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index f13fd3fc630..d3996f504a5 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -9,10 +9,12 @@ Use this to route requests between Teams """ import re -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import replace from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload + +from typing_extensions import ReadOnly from litellm._logging import verbose_logger from litellm.constants import ( @@ -56,6 +58,10 @@ class _DeploymentLike(Protocol): def get(self, key: Literal["model_name"], /) -> object: ... +class _DeploymentWithModelInfo(TypedDict, total=False): + model_info: ReadOnly[Mapping[str, object]] + + class _MetadataLike(Protocol): @overload def get(self, key: Literal["tags"], /) -> Sequence[str] | None: ... @@ -285,13 +291,14 @@ def _unknown_required_tag_hides_an_answer( def _chain_allows_fail_open( healthy_deployments: _DeploymentPool, + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], excluded_set: frozenset[str], required_set: frozenset[str], routing_confirmed: frozenset[str], ) -> bool: if _unknown_required_tag_hides_an_answer(healthy_deployments, excluded_set, required_set, routing_confirmed): return False - return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in healthy_deployments) + return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in request_deployments) def _trusted_only_pool( @@ -327,6 +334,7 @@ def _trusted_only_pool( def _resolve_or_fail_open( + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], pool: Sequence[_DeploymentLike], healthy_deployments: _DeploymentPool, excluded_set: frozenset[str], @@ -339,7 +347,7 @@ def _resolve_or_fail_open( ) -> tuple[_DeploymentLike, ...]: if pool: return tuple(pool) - if _chain_allows_fail_open(healthy_deployments, excluded_set, required_set, routing_confirmed): + if _chain_allows_fail_open(healthy_deployments, request_deployments, excluded_set, required_set, routing_confirmed): # Fall open only within whatever still satisfies whichever constraints # trace back to key/team policy. A constraint with no inherited backing at # all (or, when inherited_tags is unavailable, any constraint at all) can @@ -356,6 +364,7 @@ def _resolve_or_fail_open( def _resolve_constraint_only_pool( + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], healthy_deployments: _DeploymentPool, excluded_set: frozenset[str], required_set: frozenset[str], @@ -371,6 +380,7 @@ def _resolve_constraint_only_pool( else _exclude_deployments(_default_tagged_pool(healthy_deployments), excluded_set) ) return _resolve_or_fail_open( + request_deployments, pool, healthy_deployments, excluded_set, @@ -553,6 +563,8 @@ async def get_deployments_for_tag( healthy_deployments: _DeploymentPool, request_kwargs: _RequestKwargsLike | None = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata", + *, + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], ) -> _DeploymentPool: """ Returns a list of deployments that match the requested model and tags in the request. @@ -624,6 +636,7 @@ async def get_deployments_for_tag( if constraint_only: return _resolve_constraint_only_pool( + request_deployments, healthy_deployments, excluded_set, required_set, @@ -675,6 +688,7 @@ async def get_deployments_for_tag( if len(new_healthy_deployments) == 0 and len(default_deployments) == 0: return _resolve_or_fail_open( + request_deployments, (), healthy_deployments, excluded_set, @@ -692,6 +706,7 @@ async def get_deployments_for_tag( and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed) ): return _resolve_or_fail_open( + request_deployments, (), healthy_deployments, excluded_set, diff --git a/litellm/router_utils/baseline_request.py b/litellm/router_utils/baseline_request.py index 2f6262197a1..59878421068 100644 --- a/litellm/router_utils/baseline_request.py +++ b/litellm/router_utils/baseline_request.py @@ -7,7 +7,7 @@ from typing import Final, cast from pydantic import JsonValue, TypeAdapter, ValidationError -from litellm.llms.anthropic.pass_through.messages.utils import anthropic_messages_optional_param_keys +from litellm.llms.anthropic.messages.native.utils import anthropic_messages_optional_param_keys CACHE_SETTINGS: Final = ( "system", diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index ae913882968..2d2f5072544 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -377,7 +377,7 @@ def stream_success( end: datetime.datetime, first_chunk: datetime.datetime | None, ) -> None: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ, ) from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 1024141d5a2..017ff65e2a7 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -10,7 +10,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout -from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled +from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.rust_bridge import failures from litellm.rust_bridge.public_call import known_params, optional_str from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse @@ -64,7 +64,7 @@ def body_params(request: Mapping[str, object]) -> Mapping[str, object]: def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( anthropic_messages_stream_hidden_params, ) diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index b53e80805a5..5f75858d350 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -73,7 +73,7 @@ def _resolved_provider(request: NativeCall) -> str | None: def intercepted(request: NativeCall) -> bool: """The async Messages handler delegates matching tools to a registered interceptor.""" - from litellm.llms.anthropic.pass_through.messages.interceptors import get_messages_interceptors + from litellm.llms.anthropic.messages.native.interceptors import get_messages_interceptors tools: Final = optional_sequence(request.resolved.get("tools")) if not tools: @@ -85,6 +85,6 @@ def intercepted(request: NativeCall) -> bool: def safeguards_classifier(request: NativeCall) -> bool: """Python's proxy classifier evaluates dangerous tool use for non-Claude models.""" - from litellm.llms.anthropic.pass_through.safeguards import native_route_classifies + from litellm.llms.anthropic.messages.safeguards import native_route_classifies return native_route_classifies(str(request.resolved["model"]), request.resolved.get("safeguards")) diff --git a/litellm/utils.py b/litellm/utils.py index 1aed15ba83e..6f0b1ca2e3e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9610,12 +9610,18 @@ class ProviderConfigManager: return litellm.DatabricksDecisionsConfig() if provider == LlmProviders.HOSTED_VLLM: return litellm.HostedVLLMDecisionsConfig() + if provider == LlmProviders.LM_STUDIO: + return litellm.LMStudioDecisionsConfig() if provider == LlmProviders.LAYA: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() - if provider == LlmProviders.OPENAI: - return litellm.OpenAIDecisionsConfig() + if provider in (LlmProviders.OPENAI, LlmProviders.AZURE): + configs: Final = { + LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, + LlmProviders.AZURE: litellm.AzureDecisionsConfig, + } + return configs[provider]() if provider == LlmProviders.AZURE_AI: return litellm.AzureAIDecisionsConfig() return None @@ -9662,7 +9668,7 @@ class ProviderConfigManager: return BedrockMantleAnthropicMessagesConfig() elif litellm.LlmProviders.VERTEX_AI == provider: if "claude" in model_lower: - from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) diff --git a/ruff-strict.toml b/ruff-strict.toml index 398cca15df2..e9db4d95cc8 100644 --- a/ruff-strict.toml +++ b/ruff-strict.toml @@ -71,7 +71,7 @@ max-args = 5 # directly, each with a `# noqa: TID251 # `. "litellm.responses.main.responses".msg = "Import litellm.responses.dispatch.responses so the call routes through dispatch." "litellm.responses.main.aresponses".msg = "Import litellm.responses.dispatch.aresponses so the call routes through dispatch." -"litellm.llms.anthropic.pass_through.messages.handler.anthropic_messages".msg = "Import litellm.messages.anthropic_messages so the call routes through dispatch." -"litellm.llms.anthropic.pass_through.messages.handler.anthropic_messages_handler".msg = "Import litellm.messages.anthropic_messages_handler so the call routes through dispatch." +"litellm.llms.anthropic.messages.native.handler.anthropic_messages".msg = "Import litellm.messages.anthropic_messages so the call routes through dispatch." +"litellm.llms.anthropic.messages.native.handler.anthropic_messages_handler".msg = "Import litellm.messages.anthropic_messages_handler so the call routes through dispatch." "litellm.main.completion".msg = "Import litellm.completion so the call routes through dispatch." "litellm.main.acompletion".msg = "Import litellm.acompletion so the call routes through dispatch." diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index ea095d4510c..c3c5330c440 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -76,8 +76,8 @@ - {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven} - {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} - {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} -- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} -- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} +- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven} +- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven} - {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"} - {id: llm.responses.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.9 / LIT-4778", rationale: "Responses missing/empty input and missing model are rejected"} - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"} diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index df554b20204..cbaf46130d6 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -41,7 +41,7 @@ - {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"} - {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"} - {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"} -- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} +- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} - {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"} - {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"} - {id: quota_management.spend_tracking.discount_config.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: discount_config, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/management_endpoints/cost_tracking_settings.py + cost_calculator.py", rationale: "A persisted provider discount changes the real request's spend row"} @@ -62,11 +62,11 @@ - {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"} - {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"} - {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"} -- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"} +- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"} - {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} - {id: quota_management.spend_tracking.service_tier_stream.records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", fail_before_fix: proven, rationale: "A streamed call with no service_tier requested bills at the rates of the tier OpenAI stamps on its chunks and records that served tier on the row; the reassembled stream dropped the provider tier so the row recorded none and priced at the default rates"} - {id: quota_management.spend_tracking.service_tier_stream.responses_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [responses], source: "responses/streaming_iterator.py", rationale: "A streamed /v1/responses call bills at the tier carried on the response.completed event's inner response and records that served tier on the spend row"} -- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/pass_through/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"} +- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/messages/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"} - {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} - {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"} - {id: quota_management.spend_tracking.websearch_interception.bills_under_request_session, module: quota_management, tier: P1, behavior: spend_tracking, variant: websearch_interception, assertions: [bills_under_request_session], exercised_on: [messages], source: "integrations/websearch_interception/handler.py", fail_before_fix: proven, rationale: "A web_search server tool the proxy intercepts into litellm.asearch writes its own asearch spend row, and that row carries the parent request's session_id so the session view counts the search and its cost next to the turn that triggered it (LIT-8063)"} diff --git a/tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py b/tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py index 8842e04db6b..89a388423e0 100644 --- a/tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py +++ b/tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py @@ -14,7 +14,7 @@ from integration._support.ports import allocate_port from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter -from litellm.llms.anthropic.pass_through.adapters.transformation import truncate_tool_name +from litellm.llms.anthropic.messages.adapters.transformation import truncate_tool_name _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _JSON_OBJECTS: Final = TypeAdapter(list[dict[str, JsonValue]]) diff --git a/tests/test_litellm_rust/messages/test_body_hooks.py b/tests/test_litellm_rust/messages/test_body_hooks.py index 188cd58e00e..59f27d5dc04 100644 --- a/tests/test_litellm_rust/messages/test_body_hooks.py +++ b/tests/test_litellm_rust/messages/test_body_hooks.py @@ -8,8 +8,8 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.anthropic.pass_through.messages import interceptors -from litellm.llms.anthropic.pass_through.messages.interceptors.base import MessagesInterceptor +from litellm.llms.anthropic.messages.native import interceptors +from litellm.llms.anthropic.messages.native.interceptors.base import MessagesInterceptor from tests.test_litellm_rust.messages.data import BODY_HOOKS, MESSAGES_RESPONSE, REQUEST, call from tests.test_litellm_rust.support.callback_recorder import drain_logging from tests.test_litellm_rust.support.body_hooks import LegacyAgenticLoop, PlannedAgenticLoop, agentic_hooks diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 50f71569d73..9430e01ea8a 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -8,7 +8,7 @@ import litellm from litellm.caching.caching import Cache from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger -from litellm.llms.anthropic.pass_through.messages import handler as python_messages +from litellm.llms.anthropic.messages.native import handler as python_messages from litellm.messages import dispatch from litellm.messages.dispatch import ( _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 5b32008b279..0bc18080ef4 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -38,6 +38,7 @@ from litellm.types.decisions import ( from litellm.types.router import ModelInfo from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager +from tests.unit.conftest import AsyncOnlyImageFetch _QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( { @@ -682,7 +683,9 @@ _PPLX_USAGE: Final = {"model": "pplx-decider-v1-27b", "usage": {"input_tokens": ("reply", "detail"), ( pytest.param(_PPLX_USAGE, "DecisionsResponse answers: Field required", id="usage"), - pytest.param({"error": "Rate limit reached"}, "DecisionsResponse answers: Field required", id="rate_limit_text"), + pytest.param( + {"error": "Rate limit reached"}, "DecisionsResponse answers: Field required", id="rate_limit_text" + ), pytest.param({"error": "Request timed out"}, "DecisionsResponse answers: Field required", id="timeout_text"), pytest.param( {**_PPLX_USAGE, "answers": {"rate_limit": None}}, @@ -694,7 +697,9 @@ _PPLX_USAGE: Final = {"model": "pplx-decider-v1-27b", "usage": {"input_tokens": "DecisionsResponse answers.is_defect", id="answer_type_timeout_text", ), - pytest.param(["Rate limit reached"], "DecisionsResponse Input should be a valid dictionary", id="not_an_object"), + pytest.param( + ["Rate limit reached"], "DecisionsResponse Input should be a valid dictionary", id="not_an_object" + ), ), ) async def test_a_malformed_upstream_reply_is_a_server_error_whatever_its_text( @@ -1525,7 +1530,13 @@ async def test_databricks_router_deployment_sends_its_own_key_to_its_own_base( _PROVIDERS_WHOSE_CHAT_MODELS_ALSO_DECIDE: Final = frozenset( - {LlmProviders.OPENAI.value, LlmProviders.OPENROUTER.value, LlmProviders.HOSTED_VLLM.value} + { + LlmProviders.OPENAI.value, + LlmProviders.AZURE.value, + LlmProviders.OPENROUTER.value, + LlmProviders.HOSTED_VLLM.value, + LlmProviders.LM_STUDIO.value, + } ) _DEDICATED_DECIDER_PROVIDERS: Final[tuple[str, ...]] = tuple( sorted( @@ -3791,7 +3802,12 @@ async def test_a_returned_model_registered_without_prices_is_billed_at_the_reque monkeypatch.setitem( litellm.model_cost, "typesafe/jev-test-alias", - {"litellm_provider": "typesafe", "mode": "evaluation", "input_cost_per_token": 0.001, "output_cost_per_token": 0}, + { + "litellm_provider": "typesafe", + "mode": "evaluation", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0, + }, ) monkeypatch.setitem(litellm.model_cost, "typesafe/jev-test-unpriced", unpriced_row) respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json={**_RESPONSE, "model": "jev-test-unpriced"}) @@ -3893,3 +3909,120 @@ async def test_native_decisions_distinguishes_missing_usage_from_reported_zero( expected = litellm.completion_cost(completion_response=response) assert response._hidden_params["response_cost"] == pytest.approx(expected) assert (expected == 0) == (usage["input_tokens"] == usage["output_tokens"] == 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", (True, False), ids=("asystemone", "systemone")) +@pytest.mark.parametrize("api_base", ("http://lmstudio.local:1234", "http://lmstudio.local:1234/v1")) +async def test_lm_studio_systemone_preserves_questions_and_omits_placeholder_auth( + use_async: bool, api_base: str, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key") + route: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + kwargs: Final = { + "model": "lm_studio/local-decision", + "state": "review", + "questions": _QUESTIONS, + "api_base": api_base, + } + + response: Final = await litellm.asystemone(**kwargs) if use_async else litellm.systemone(**kwargs) + + assert json.loads(route.calls.last.request.content) == { + "model": "local-decision", + "state": "review", + "questions": dict(_QUESTIONS), + } + assert "authorization" not in route.calls.last.request.headers + assert response.answers["is_defect"] == NoulAnswer(type="noul", noul=0.9) + assert response.answers["sentiment"].choice == "positive" + assert response.answers["severity"].score == 1 + assert response.usage == DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", (True, False), ids=("adecisions", "decisions")) +async def test_lm_studio_decisions_translates_openai_questions_and_answers( + use_async: bool, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) + route: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + kwargs: Final = { + "model": "lm_studio/local-decision", + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + "api_base": "http://lmstudio.local:1234/v1", + } + + response: Final = await litellm.adecisions(**kwargs) if use_async else litellm.decisions(**kwargs) + + assert json.loads(route.calls.last.request.content) == { + "model": "local-decision", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers[0].type == "predicate" + assert response.answers[0].name == "is_defect" + assert response.answers[0].probability == 0.9 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("explicit_key", (None, "deployment-key")) +async def test_lm_studio_decisions_resolves_environment_connection_and_explicit_key_override( + explicit_key: str | None, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setenv("LM_STUDIO_API_BASE", "http://lmstudio.local:1234/v1") + monkeypatch.setenv("LM_STUDIO_API_KEY", "environment-key") + route: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + + response: Final = await litellm.asystemone( + model="lm_studio/local-decision", state="review", questions=_QUESTIONS, api_key=explicit_key + ) + + assert route.calls.last.request.headers["authorization"] == f"Bearer {explicit_key or 'environment-key'}" + assert response.answers["is_defect"] == NoulAnswer(type="noul", noul=0.9) + + +@pytest.mark.asyncio +async def test_lm_studio_decisions_requires_a_server_before_http( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.delenv("LM_STUDIO_API_BASE", raising=False) + with pytest.raises(litellm.BadRequestError, match="api_base is required"): + await litellm.asystemone(model="lm_studio/local-decision", state="review", questions=_QUESTIONS) + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("remote_image", (False, True), ids=("inline", "remote")) +async def test_lm_studio_decisions_sends_inline_images_to_systemone( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + remote_image: bool, + async_only_image_fetch: AsyncOnlyImageFetch, +) -> None: + monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) + image: Final = "https://images.example/input.png" if remote_image else "data:image/png;base64,AA==" + expected_image: Final = async_only_image_fetch.data_url if remote_image else image + route: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="lm_studio/local-decision", + input=[ + { + "role": "user", + "content": [{"type": "input_text", "text": "review"}, {"type": "input_image", "image_url": image}], + } + ], + questions=[{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + api_base="http://lmstudio.local:1234/v1", + ) + + assert json.loads(route.calls.last.request.content) == { + "model": "local-decision", + "state": "review", + "images": [expected_image], + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert response.answers[0].probability == 0.9 diff --git a/tests/unit/images/test_main.py b/tests/unit/images/test_main.py index ff43e892eab..38af99ff505 100644 --- a/tests/unit/images/test_main.py +++ b/tests/unit/images/test_main.py @@ -1,6 +1,8 @@ import asyncio +import base64 import json from datetime import datetime +from pathlib import Path from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -405,3 +407,56 @@ async def test_azure_image_generation_request_body(): call_args = mock_post.call_args request_json = call_args.kwargs.get("json", {}) assert request_json == expected_body + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("transport", ["headers", "extra_headers", "scoped", "api_key"]) +@pytest.mark.parametrize("saved_key", [None, "fake-saved-gateway-key"]) +async def test_chatgpt_images_forward_client_login_and_workspace( + asynchronous: bool, + transport: str, + saved_key: str | None, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path)) + claims: Final = {"https://api.openai.com/auth": {"chatgpt_account_id": "client-account"}} + payload: Final = base64.urlsafe_b64encode(json.dumps(claims).encode()).decode().rstrip("=") + token: Final = f"test.{payload}.test" + headers: Final = {"aUtHoRiZaTiOn": f"Bearer {token}", "ChatGPT-Account-Id": "selected-workspace"} + credentials: Final = ( + {"provider_specific_header": {"custom_llm_provider": "chatgpt", "extra_headers": headers}} + if transport == "scoped" + else {"api_key": token, "extra_headers": {"cHaTgPt-AcCoUnT-Id": "selected-workspace"}} + if transport == "api_key" + else {transport: {**headers, "cookie": "fake-cookie", "x-litellm-api-key": "fake-gateway-key"}} + ) + route: Final = respx_mock.post("https://chatgpt.com/backend-api/codex/images/generations").mock( + return_value=httpx.Response(200, json={"created": 1700000000, "data": [{"b64_json": "aW1n"}]}) + ) + params: Final = { + "model": "chatgpt/gpt-image-2", + "prompt": "A blue circle", + "api_key": saved_key, + **( + { + "client": AsyncOpenAI( + api_key="fake", base_url="https://chatgpt.com/backend-api/codex", http_client=httpx.AsyncClient() + ) + } + if asynchronous + else {} + ), + **credentials, + } + response: Final = await litellm.aimage_generation(**params) if asynchronous else litellm.image_generation(**params) + assert response.data[0].b64_json == "aW1n" + request: Final = route.calls.last.request + assert request.headers["authorization"] == f"Bearer {token}" + assert request.headers["chatgpt-account-id"] == "selected-workspace" + assert request.headers.get_list("content-type") == ["application/json"] + assert "cookie" not in request.headers + assert "x-litellm-api-key" not in request.headers + assert json.loads(request.content) == {"model": "gpt-image-2", "prompt": "A blue circle"} diff --git a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 64c49f03732..351ca9beb4d 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -25,7 +25,7 @@ from litellm.integrations.websearch_interception.handler import ( ) from litellm.integrations.websearch_interception.tools import get_litellm_web_search_tool from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS -from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( +from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult diff --git a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py index 4fbc78c2f4d..fdc3771e903 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py @@ -238,7 +238,7 @@ class TestShortCircuitEntryPoint: @pytest.mark.asyncio async def test_returns_none_when_no_callbacks(self): """No callbacks configured → returns None""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -255,7 +255,7 @@ class TestShortCircuitEntryPoint: @pytest.mark.asyncio async def test_returns_dict_when_not_streaming(self): """Non-streaming short-circuit → returns dict""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -280,10 +280,10 @@ class TestShortCircuitEntryPoint: @pytest.mark.asyncio async def test_returns_stream_iterator_when_streaming(self): """Streaming short-circuit → returns FakeAnthropicMessagesStreamIterator""" - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -322,7 +322,7 @@ class TestShortCircuitEntryPoint: """Non-WebSearchInterceptionLogger callbacks are ignored""" from unittest.mock import MagicMock - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -345,10 +345,10 @@ class TestShortCircuitEntryPoint: loop. The short-circuit must use the ORIGINAL stream value so streaming callers get SSE events instead of a plain dict. """ - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -378,7 +378,7 @@ class TestShortCircuitEntryPoint: still fire the short-circuit when the caller propagates the derived provider. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) @@ -410,7 +410,7 @@ class TestShortCircuitSafeguardResults: NO_TOOL_USES_AVAILABLE = [{"type": "dangerous_tool_use", "status": {"type": "available", "tool_uses": {}}}] async def _short_circuit(self, *, stream: bool, kwargs: dict | None): - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( _try_websearch_short_circuit, ) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_streaming_wrap.py b/tests/unit/integrations/websearch_interception/test_websearch_streaming_wrap.py index f4a46efaa1d..1401f260427 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_streaming_wrap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_streaming_wrap.py @@ -13,7 +13,7 @@ import pytest from litellm.integrations.custom_logger import CustomLogger from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( +from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) from litellm.types.integrations.custom_logger import AgenticLoopPlan diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 561ac9b2b33..b46c65d4460 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6575,7 +6575,7 @@ def test_handle_anthropic_messages_response_logging_keeps_the_served_response_id def test_handle_anthropic_messages_response_logging_bills_the_bridged_chat_completion_not_the_served_body(): """A chat deployment answers /v1/messages through the chat bridge. The Anthropic body served to the caller has no reasoning token field, so the spend row must come from the recorded chat completion.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.types.utils import CompletionTokensDetailsWrapper, Usage chat_completion = ModelResponse( diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 88af656ad2e..f222a257b8f 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -3,6 +3,7 @@ import asyncio import base64 import importlib +import io import json import os import subprocess @@ -19,6 +20,7 @@ from unittest.mock import MagicMock, sentinel import anyio.to_thread import pytest import tiktoken +from PIL import Image from tokenizers import Regex, Tokenizer, models, pre_tokenizers from unittest.mock import AsyncMock, patch @@ -39,7 +41,7 @@ from litellm.litellm_core_utils.token_counter import ( offload_token_count, ) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new -from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, @@ -2015,6 +2017,71 @@ def _png_data_url(width: int, height: int) -> str: return "data:image/png;base64," + base64.b64encode(ihdr + b"\x08\x06\x00\x00\x00").decode() +@pytest.mark.parametrize( + ("width", "height", "expected"), [(256, 256, 262), (512, 512, 262), (768, 768, 772), (1024, 1024, 772)] +) +@pytest.mark.parametrize("detail", [None, "auto", "high"]) +def test_inline_auto_images_count_high_detail_tiles(width: int, height: int, expected: int, detail: str | None) -> None: + image_url = {"url": _png_data_url(width, height)} + if detail is not None: + image_url["detail"] = detail + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": image_url}]}] + + assert token_counter_new(model="gpt-4o", messages=messages) == expected + + +@pytest.mark.parametrize("image_format", ["PNG", "JPEG", "GIF", "WEBP"]) +def test_inline_auto_images_read_supported_formats(image_format: str) -> None: + output = io.BytesIO() + Image.new("RGB", (1024, 1024)).save(output, format=image_format) + url = f"data:image/{image_format.lower()};base64," + base64.b64encode(output.getvalue()).decode() + + assert calculate_img_tokens(url) == 765 + + +@pytest.mark.parametrize( + "data", + [ + "https://example.com/image.png", + "http://127.0.0.1/image.png", + "http://169.254.169.254/latest/meta-data/", + "file:///etc/passwd", + "data:application/pdf;base64,JVBERi0xLjQK", + "data:image/png;base64,%%%", + "data:image/png;base64,iVBORw0KGgo=", + "data:image/jpeg;base64,/9j/4AAQ", + "data:image/png,not-base64", + "", + _png_data_url(0, 1024), + _png_data_url(1024, 0), + ], +) +def test_auto_image_fallback_does_not_fetch_urls_or_raise(data: str) -> None: + # The unit suite disables sockets; a fetch must also not be attempted and swallowed. + with patch("httpx.Client.get", side_effect=AssertionError("auto counting must not fetch URLs")) as get: + assert calculate_img_tokens(data) == 85 + get.assert_not_called() + + +def test_auto_image_decode_respects_size_limit(monkeypatch: pytest.MonkeyPatch) -> None: + counter = importlib.import_module("litellm.litellm_core_utils.token_counter") + monkeypatch.setattr(counter, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 24 / (1024 * 1024)) + + with patch("base64.b64decode", side_effect=AssertionError("oversized input must not be decoded")) as decode: + assert calculate_img_tokens(_png_data_url(1024, 1024)) == 85 + decode.assert_not_called() + + +def test_low_and_default_image_counts_skip_inline_decoding() -> None: + with patch("base64.b64decode", side_effect=AssertionError("fixed counts must not decode")) as decode: + assert calculate_img_tokens(_png_data_url(1024, 1024), mode="low") == 85 + assert ( + calculate_img_tokens(_png_data_url(1024, 1024), use_default_image_token_count=True) + == litellm.constants.DEFAULT_IMAGE_TOKEN_COUNT + ) + decode.assert_not_called() + + @pytest.mark.parametrize(("width", "height"), [(1, 1), (768, 768), (2000, 768), (768, 2000), (4096, 4096), (8000, 3072)]) def test_high_detail_image_token_upper_bound_covers_every_image_size(width: int, height: int) -> None: assert calculate_img_tokens(_png_data_url(width, height), mode="high") <= high_detail_image_token_upper_bound() diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 8218c120153..9e88b59adc9 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -18,7 +18,7 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import Streami from litellm.llms.anthropic.chat.guardrail_translation.handler import ( AnthropicMessagesHandler, ) -from litellm.llms.anthropic.pass_through.adapters.transformation import truncate_tool_name +from litellm.llms.anthropic.messages.adapters.transformation import truncate_tool_name from litellm.types.utils import ChatCompletionMessageToolCall, GenericGuardrailAPIInputs diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index b3b04440d16..ef002a73e52 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -26,7 +26,7 @@ from litellm.llms.anthropic.chat import ModelResponseIterator from litellm.llms.anthropic.common_utils import process_anthropic_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig @@ -6152,7 +6152,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): [ ("litellm.llms.anthropic.chat.transformation", "AnthropicConfig", False), ( - "litellm.llms.anthropic.pass_through.messages.transformation", + "litellm.llms.anthropic.messages.native.transformation", "AnthropicMessagesConfig", False, ), @@ -6194,7 +6194,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke(): True, ), ( - "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation", + "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation", "VertexAIPartnerModelsAnthropicMessagesConfig", True, ), diff --git a/tests/unit/llms/anthropic/pass_through/adapters/__init__.py b/tests/unit/llms/anthropic/messages/adapters/__init__.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/adapters/__init__.py rename to tests/unit/llms/anthropic/messages/adapters/__init__.py diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py rename to tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index 658aa009683..4e4b89b764b 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -21,14 +21,14 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, _bedrock_converse_messages_pt, ) -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, AnthropicAdapter, LiteLLMAnthropicMessagesAdapter, create_tool_name_mapping, truncate_tool_name, ) -from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( +from litellm.llms.anthropic.messages.native.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, ) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -3957,7 +3957,7 @@ class TestAnthropicStreamWrapperToolArgs: return [text_chunk, tool_chunk, finish_chunk] def _make_stream_wrapper(self, chunks): - from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( + from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) @@ -4072,7 +4072,7 @@ def _make_simple_openai_response( def test_translate_openai_response_to_anthropic_with_polyfill_compaction_block(): """compaction_block from PolyfillResult must be prepended to content at index 0.""" - from litellm.llms.anthropic.pass_through.context_management.result import ( + from litellm.llms.anthropic.messages.context_management.result import ( PolyfillResult, ) @@ -4105,7 +4105,7 @@ def test_translate_openai_response_to_anthropic_with_polyfill_compaction_block() def test_translate_openai_response_to_anthropic_with_polyfill_iterations_usage(): """iterations_usage from PolyfillResult must produce usage['iterations'] with a message entry.""" - from litellm.llms.anthropic.pass_through.context_management.result import ( + from litellm.llms.anthropic.messages.context_management.result import ( PolyfillResult, ) @@ -4160,7 +4160,7 @@ def test_translate_openai_response_to_anthropic_no_polyfill_no_change(): def test_translate_openai_response_to_anthropic_with_polyfill_both_compaction_and_iterations(): """Full summary path: compaction_block and iterations_usage both present simultaneously.""" - from litellm.llms.anthropic.pass_through.context_management.result import ( + from litellm.llms.anthropic.messages.context_management.result import ( PolyfillResult, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_context_management.py similarity index 95% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_context_management.py index 9b5477e3206..81cc1fbbe02 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_context_management.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_context_management.py @@ -5,8 +5,8 @@ from unittest.mock import patch import pytest import litellm -from litellm.llms.anthropic.pass_through.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler -from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError +from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError from litellm.router import Router from litellm.types.utils import Choices, Message, ModelResponse, Usage diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_output_config_passthrough.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_output_config_passthrough.py index 03553e65ad4..c34cb686928 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_output_config_passthrough.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_output_config_passthrough.py @@ -37,7 +37,7 @@ import pytest # where pytest is invoked from. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( ANTHROPIC_ONLY_REQUEST_KEYS, LiteLLMMessagesToCompletionTransformationHandler, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py index c31d85be0a8..ed6cd6134ff 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_prompt_cache_key.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_prompt_cache_key.py @@ -6,7 +6,7 @@ import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index 5a82b9af468..e504513dd36 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -16,7 +16,7 @@ import pytest import respx import litellm -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_safeguard_results.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_safeguard_results.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/adapters/test_handler_safeguard_results.py rename to tests/unit/llms/anthropic/messages/adapters/test_handler_safeguard_results.py index 51964a77b67..cbe65b8a4e0 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_safeguard_results.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_safeguard_results.py @@ -11,11 +11,11 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, request_safeguards_evaluator, ) -from litellm.llms.anthropic.pass_through.safeguards import ToolUseUnderReview +from litellm.llms.anthropic.messages.safeguards import ToolUseUnderReview from litellm.proxy._types import LiteLLM_TeamTableCachedObj, UserAPIKeyAuth from litellm.router import Router from litellm.types.utils import ( @@ -30,7 +30,7 @@ from litellm.types.utils import ( StreamingChoices, Usage, ) -from tests.unit.llms.anthropic.pass_through.conftest import ProxyAuthState +from tests.unit.llms.anthropic.messages.conftest import ProxyAuthState SAFEGUARDS: Final = [{"type": "dangerous_tool_use", "classifier_context": {"permission_mode": "auto"}}] BASH_TOOL: Final = {"name": "Bash", "description": "run", "input_schema": {"type": "object", "properties": {}}} diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py index 7400cd08dd3..d4911427e73 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_combined_chunk.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py @@ -22,7 +22,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo convert_to_streaming_response, convert_to_streaming_response_async, ) -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, _CombinedChunkSplitter, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_compaction.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_compaction.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_compaction.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_compaction.py index 2610824edf4..1d60f71e196 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_compaction.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_compaction.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import Delta, StreamingChoices, Usage diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_empty_choices.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_empty_choices.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_empty_choices.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_empty_choices.py index 66825124f25..826c16f6b97 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_empty_choices.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_empty_choices.py @@ -12,7 +12,7 @@ import asyncio import json from typing import Any, AsyncIterator, Dict, List, Optional -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_first_delta.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_first_delta.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py index 92234282bb2..32ec0ffae8e 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_first_delta.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_first_delta.py @@ -27,7 +27,7 @@ from unittest.mock import MagicMock import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import ( diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_message_id.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py similarity index 96% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_message_id.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py index a4f851c7753..b98877f25aa 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_message_id.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_message_id.py @@ -12,10 +12,10 @@ import pytest import respx import litellm -from litellm.llms.anthropic.pass_through.adapters.handler import ( +from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py index 6951253349e..2cc0d86a3a8 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_mid_stream_error.py @@ -36,7 +36,7 @@ from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, _mid_stream_error_sse_event, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_sse_stream.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_sse_stream.py index ec57636a17b..79976564e5a 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_sse_stream.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_sse_stream.py @@ -13,7 +13,7 @@ import pytest from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicSSEStream, AnthropicStreamWrapper, ) diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_stop_reason.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_stop_reason.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_stop_reason.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_stop_reason.py index 5f5007d52b5..48ffad5a050 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_stop_reason.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_stop_reason.py @@ -12,7 +12,7 @@ bridge emitted ``stop_reason: "end_turn"`` and Anthropic tool-runners import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.llms.ollama.chat.transformation import ( diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_tool_args.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_tool_args.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_tool_args.py rename to tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_tool_args.py index 224775ad5b9..d34a978f228 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_tool_args.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_tool_args.py @@ -17,7 +17,7 @@ from unittest.mock import MagicMock import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import ( diff --git a/tests/unit/llms/anthropic/pass_through/conftest.py b/tests/unit/llms/anthropic/messages/conftest.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/conftest.py rename to tests/unit/llms/anthropic/messages/conftest.py diff --git a/tests/unit/llms/anthropic/pass_through/context_management/__init__.py b/tests/unit/llms/anthropic/messages/context_management/__init__.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/context_management/__init__.py rename to tests/unit/llms/anthropic/messages/context_management/__init__.py diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py rename to tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py index 2ffa73dda21..2567d1b76db 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_clear_tool_uses.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_clear_tool_uses.py @@ -10,17 +10,17 @@ from unittest.mock import AsyncMock, MagicMock import pytest import litellm -from litellm.llms.anthropic.pass_through.adapters.handler import _run_polyfill_if_enabled -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.adapters.handler import _run_polyfill_if_enabled +from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, PolyfillResult, apply_context_management, ) -from litellm.llms.anthropic.pass_through.context_management.constants import ( +from litellm.llms.anthropic.messages.context_management.constants import ( CLEARED_TOOL_RESULT_PLACEHOLDER, COMPACT_SUMMARY_MODEL_SETTING_KEY, ) -from litellm.llms.anthropic.pass_through.context_management.editors.clear_tool_uses import ( +from litellm.llms.anthropic.messages.context_management.editors.clear_tool_uses import ( apply_clear_tool_uses_20250919, ) from litellm.types.llms.anthropic import AppliedEdit diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py b/tests/unit/llms/anthropic/messages/context_management/test_compact.py similarity index 91% rename from tests/unit/llms/anthropic/pass_through/context_management/test_compact.py rename to tests/unit/llms/anthropic/messages/context_management/test_compact.py index 9b3084f0e7c..159c3586c53 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_compact.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_compact.py @@ -21,11 +21,11 @@ import pytest from fastapi import HTTPException import litellm -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( AnthropicContextManagementError, apply_context_management, ) -from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( +from litellm.llms.anthropic.messages.context_management.editors.compact import ( _augment_system_with_summary, _extract_summary_text, _select_last_user_question, @@ -34,7 +34,7 @@ from litellm.llms.anthropic.pass_through.context_management.editors.compact impo apply_client_compaction_block_history, apply_compact_20260112, ) -from litellm.llms.anthropic.pass_through.context_management.result import ( +from litellm.llms.anthropic.messages.context_management.result import ( PolyfillResult, ) from litellm.proxy._types import ( @@ -46,7 +46,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.hooks.parallel_request_limiter_v3 import RateLimitUnverifiableError -from tests.unit.llms.anthropic.pass_through.conftest import ProxyAuthState +from tests.unit.llms.anthropic.messages.conftest import ProxyAuthState MODEL = "openai/gpt-4o" @@ -321,7 +321,7 @@ async def test_trigger_below_minimum_raises(): async def test_trigger_at_minimum_does_not_raise(): """Exactly 50 000 is allowed — only strictly less than 50k is rejected.""" with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -346,7 +346,7 @@ async def test_trigger_at_minimum_does_not_raise(): async def test_opt_in_gating_no_summary_model_configured(): messages = _simple_messages() with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -373,7 +373,7 @@ async def test_opt_in_gating_no_summary_model_keeps_post_compaction_tail(): messages = _messages_with_compaction("prior summary text") with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -455,7 +455,7 @@ async def test_slice_only_path_with_existing_compaction_block(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=500), # well under threshold @@ -492,7 +492,7 @@ async def test_slice_only_no_compaction_block_under_threshold(): messages = _simple_messages() with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=500), @@ -523,12 +523,12 @@ async def test_full_summary_path(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), # over 150k threshold patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ), @@ -577,7 +577,7 @@ async def test_full_summary_path_uses_router_when_available(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="my-summary-model", ), patch("litellm.token_counter", return_value=200_000), @@ -613,12 +613,12 @@ async def test_litellm_metadata_propagated_to_summary_call(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ) as mock_call, @@ -650,12 +650,12 @@ async def test_summary_call_failed(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, side_effect=RuntimeError("network error"), ), @@ -686,12 +686,12 @@ async def test_summary_extraction_failed_no_tags(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ), @@ -718,7 +718,7 @@ async def test_pause_after_compaction_ignored_warning(): """pause_after_compaction: true → warning recorded, request proceeds normally.""" messages = _simple_messages() with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -741,7 +741,7 @@ async def test_pause_after_compaction_ignored_warning(): async def test_unsupported_trigger_type_falls_back_to_default(): messages = _simple_messages() with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_compact_20260112( @@ -779,12 +779,12 @@ async def test_custom_instructions_used_verbatim(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -824,12 +824,12 @@ async def test_default_instructions_appended_with_no_tool_suffix_when_no_tools() with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -860,12 +860,12 @@ async def test_default_instructions_with_tools_appends_no_tool_suffix(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -894,12 +894,12 @@ async def test_system_prompt_forwarded_to_summary_call_as_string(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -934,12 +934,12 @@ async def test_system_prompt_forwarded_to_summary_call_as_content_blocks(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -977,12 +977,12 @@ async def test_summary_call_carries_prior_compaction_summary_into_system(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -1014,12 +1014,12 @@ async def test_summary_call_omits_system_message_when_system_is_none(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -1055,12 +1055,12 @@ async def test_summary_call_does_not_emit_consecutive_user_turns(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", side_effect=_fake_call_summary_model, ), ): @@ -1085,10 +1085,10 @@ async def test_summary_call_sends_default_max_tokens(): (which require it) don't reject the request and silently fall back to ``summary_call_failed``. """ - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_SUMMARY_MAX_TOKENS, ) - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -1112,7 +1112,7 @@ async def test_summary_call_sends_default_max_tokens(): async def test_summary_call_honors_max_tokens_override(): """Operators can override the default summary ``max_tokens`` via ``general_settings.context_management_summary_max_tokens``.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _read_summary_max_tokens_setting, ) @@ -1129,7 +1129,7 @@ async def test_summary_call_honors_max_tokens_override(): ): assert _read_summary_max_tokens_setting() == 8192 - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -1148,10 +1148,10 @@ def test_summary_max_tokens_setting_falls_back_for_invalid_values(): """Invalid override values (non-int, non-positive, missing) fall back to the compiled default so a typo in ``general_settings`` doesn't break the summary call.""" - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_SUMMARY_MAX_TOKENS, ) - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _read_summary_max_tokens_setting, ) @@ -1168,10 +1168,10 @@ def test_summary_max_tokens_setting_falls_back_for_invalid_values(): async def test_summary_call_sends_default_timeout(): """``timeout`` is set on the summary call so a slow or unresponsive summary model cannot hang the parent ``/v1/messages`` request indefinitely.""" - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_SUMMARY_TIMEOUT_SECONDS, ) - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -1235,12 +1235,12 @@ async def test_summary_model_denied_when_key_not_in_allowlist(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), ): @@ -1267,12 +1267,12 @@ async def test_summary_model_allowed_when_in_key_allowlist(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), ): @@ -1298,12 +1298,12 @@ async def test_summary_model_allowed_when_no_user_api_key_auth(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), ): @@ -1322,12 +1322,12 @@ async def test_summary_model_allowed_when_no_user_api_key_auth(): async def _compact_as(auth: UserAPIKeyAuth, summary_call: AsyncMock) -> PolyfillResult: with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", summary_call, ), ): @@ -1443,12 +1443,12 @@ async def test_summary_model_denied_when_key_over_model_budget(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), @@ -1494,12 +1494,12 @@ async def test_summary_model_denied_when_user_over_model_budget(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), @@ -1537,7 +1537,7 @@ async def test_summary_model_denied_when_user_over_model_budget(): async def test_summary_model_denied_when_team_member_over_model_budget(monkeypatch: pytest.MonkeyPatch): from litellm.caching.dual_cache import DualCache - from litellm.llms.anthropic.pass_through.context_management.constants import COMPACT_SUMMARY_MODEL_SETTING_KEY + from litellm.llms.anthropic.messages.context_management.constants import COMPACT_SUMMARY_MODEL_SETTING_KEY from litellm.models.team_membership import LiteLLM_TeamMembership from litellm.proxy._types import ( LiteLLM_BudgetTable, @@ -1671,12 +1671,12 @@ async def test_summary_model_denied_when_end_user_over_model_budget(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), @@ -1712,12 +1712,12 @@ async def test_summary_model_allowed_when_within_model_budget(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.model_max_budget_limiter", limiter), @@ -1803,12 +1803,12 @@ async def test_summary_model_rate_limit_check_errors(limiter_error, summary_call with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging), @@ -1845,12 +1845,12 @@ async def test_summary_model_denied_when_over_rate_limit(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging), @@ -1881,12 +1881,12 @@ async def test_summary_model_allowed_when_within_rate_limit(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging), @@ -1929,12 +1929,12 @@ async def test_summary_model_allowed_while_the_caller_holds_the_keys_only_parall with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", _proxy_logging_like_the_live_proxy(limiter)), @@ -1965,12 +1965,12 @@ async def test_summary_model_rate_limit_skipped_for_legacy_limiter(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging), @@ -2017,12 +2017,12 @@ async def test_summary_model_denied_when_team_over_model_budget(): with ( patch( # test-quality-ok: apply_compact_20260112 reads the summary model setting as a module global, no seam - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), # test-quality-ok: forces the over-threshold branch patch( # test-quality-ok: the summary call is the observable that must NOT happen when the team is over budget - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), patch( # test-quality-ok: the limiter is a proxy_server module global the editor imports, no injection seam @@ -2073,12 +2073,12 @@ async def test_scoped_budget_metadata_propagated_to_summary_call(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ) as mock_call, @@ -2101,7 +2101,7 @@ async def test_scoped_budget_metadata_propagated_to_summary_call(): async def test_summary_call_passes_end_user_id_as_top_level_user(): """``_call_summary_model`` forwards the propagated end-user id as the top-level ``user`` kwarg that legacy limiter / prometheus end-user tracking reads.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -2124,7 +2124,7 @@ async def test_summary_call_passes_end_user_id_as_top_level_user(): async def test_summary_call_omits_user_when_no_end_user_id(): """No end-user id on the parent request means no ``user`` kwarg is sent.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -2158,12 +2158,12 @@ async def test_model_budget_metadata_propagated_to_summary_call(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", new_callable=AsyncMock, return_value=mock_response, ) as mock_call, @@ -2195,12 +2195,12 @@ async def test_summary_call_propagates_allowed_model_region(): with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._call_summary_model", + "litellm.llms.anthropic.messages.context_management.editors.compact._call_summary_model", mock_call, ), ): @@ -2221,7 +2221,7 @@ async def test_summary_call_omits_allowed_model_region_when_unset(): """Callers without a region restriction must not get an ``allowed_model_region=None`` kwarg, which would otherwise force the router to evaluate region filtering. """ - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -2244,7 +2244,7 @@ async def test_summary_call_omits_allowed_model_region_when_unset(): async def test_summary_call_forwards_allowed_model_region_when_set(): """When the caller is region-restricted, the kwarg reaches the router.""" - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _call_summary_model, ) @@ -2275,7 +2275,7 @@ async def test_dispatcher_routes_compact_edit(): """compact_20260112 in the dispatcher resolves to opt-in gate when no model set.""" messages = _simple_messages() with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await apply_context_management( @@ -2317,7 +2317,7 @@ async def test_dispatcher_trigger_below_minimum_raises_through(): async def test_run_polyfill_skipped_when_context_management_in_additional_drop_params(): """additional_drop_params=["context_management"] is the explicit opt-out.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( _run_polyfill_if_enabled, ) @@ -2338,13 +2338,13 @@ async def test_run_polyfill_runs_when_litellm_drop_params_true(monkeypatch): """drop_params must not disable the polyfill: context_management is a LiteLLM-supported param (polyfilled where not native), and drop_params only exists to strip genuinely unsupported params.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( _run_polyfill_if_enabled, ) monkeypatch.setattr(litellm, "drop_params", True) with patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value=None, ): result = await _run_polyfill_if_enabled( @@ -2363,7 +2363,7 @@ async def test_run_polyfill_runs_when_litellm_drop_params_true(monkeypatch): async def test_run_polyfill_skipped_when_spec_empty(): """Empty context_management_spec must also return None (no polyfill work).""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( _run_polyfill_if_enabled, ) @@ -2444,7 +2444,7 @@ def _openai_chat_response(): async def _call_async_adapter_handler(**handler_kwargs: Any): - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -2503,7 +2503,7 @@ async def test_async_handler_additional_drop_params_strips_context_management(): def _call_sync_adapter_handler(**handler_kwargs: Any): - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -2555,7 +2555,7 @@ async def test_prepare_context_managed_request_forwards_proxy_litellm_metadata() Anthropic-shape ``metadata`` arg (which only carries ``user_id``). Otherwise the summary subcall lands on the router with no parent attribution, and those tokens go unbilled to the caller's key/team.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( prepare_context_managed_request, ) @@ -2568,7 +2568,7 @@ async def test_prepare_context_managed_request_forwards_proxy_litellm_metadata() with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="claude-haiku-4-5", ), patch("litellm.token_counter", return_value=200_000), @@ -2759,7 +2759,7 @@ def test_endpoint_runs_failure_hook_on_500_context_management_error(): def test_count_effective_tokens_counts_midturn_system_correction(): - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _count_effective_tokens, ) @@ -2786,7 +2786,7 @@ def test_count_effective_tokens_counts_midturn_system_correction(): def test_build_summary_messages_keeps_midturn_system_correction_in_place(): - from litellm.llms.anthropic.pass_through.context_management.editors.compact import ( + from litellm.llms.anthropic.messages.context_management.editors.compact import ( _build_summary_messages, ) @@ -2820,7 +2820,7 @@ async def test_threshold_check_counts_tokens_off_the_event_loop(monkeypatch): warm_tokenizer, ) - from litellm.llms.anthropic.pass_through.context_management.constants import ( + from litellm.llms.anthropic.messages.context_management.constants import ( COMPACT_SUMMARY_MODEL_SETTING_KEY, ) from litellm.proxy.proxy_server import general_settings diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_constants.py b/tests/unit/llms/anthropic/messages/context_management/test_constants.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/context_management/test_constants.py rename to tests/unit/llms/anthropic/messages/context_management/test_constants.py index 680c2b6f79a..da0d9c38791 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_constants.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_constants.py @@ -8,7 +8,7 @@ import pytest import litellm from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.context_management.constants import ( +from litellm.llms.anthropic.messages.context_management.constants import ( CLEARED_TOOL_RESULT_PLACEHOLDER, ) from litellm.types.utils import ( diff --git a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py rename to tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py index 66957456c95..b79e33b0c3b 100644 --- a/tests/unit/llms/anthropic/pass_through/context_management/test_dispatcher.py +++ b/tests/unit/llms/anthropic/messages/context_management/test_dispatcher.py @@ -2,7 +2,7 @@ Unit tests for the context_management polyfill dispatcher. """ -from litellm.llms.anthropic.pass_through.context_management import ( +from litellm.llms.anthropic.messages.context_management import ( apply_context_management, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/__init__.py b/tests/unit/llms/anthropic/messages/native/__init__.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/messages/__init__.py rename to tests/unit/llms/anthropic/messages/native/__init__.py diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_advisor_integration.py b/tests/unit/llms/anthropic/messages/native/test_advisor_integration.py similarity index 92% rename from tests/unit/llms/anthropic/pass_through/messages/test_advisor_integration.py rename to tests/unit/llms/anthropic/messages/native/test_advisor_integration.py index f5f30fb86c1..8bd36c6564a 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_advisor_integration.py +++ b/tests/unit/llms/anthropic/messages/native/test_advisor_integration.py @@ -70,7 +70,7 @@ async def test_full_dispatch_interceptor_fires_and_loop_completes(): The interceptor must fire, run the loop (1 advisor call), and return a clean final response with no advisor tool_use blocks. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) @@ -86,7 +86,7 @@ async def test_full_dispatch_interceptor_fires_and_loop_completes(): return _text_resp("def is_prime(n): ...") # executor: final with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_handler, ): result = await anthropic_messages( @@ -121,10 +121,10 @@ async def test_max_uses_enforced_through_full_handler(): AdvisorMaxIterationsError propagates out of anthropic_messages() when the executor keeps calling the advisor past max_uses. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, ) @@ -137,7 +137,7 @@ async def test_max_uses_enforced_through_full_handler(): return _advisor_call_resp() with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_handler, ): with pytest.raises(AdvisorMaxIterationsError): @@ -162,7 +162,7 @@ async def test_anthropic_provider_bypasses_interceptor(): With custom_llm_provider='anthropic', the interceptor must NOT fire. The advisor_20260301 tool is forwarded as-is to the underlying handler. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) @@ -170,7 +170,7 @@ async def test_anthropic_provider_bypasses_interceptor(): # Patch the non-interceptor code path — anthropic_messages_handler with patch( - "litellm.llms.anthropic.pass_through.messages.handler.anthropic_messages_handler", + "litellm.llms.anthropic.messages.native.handler.anthropic_messages_handler", return_value=direct_response, ) as mock_native: result = await anthropic_messages( @@ -209,7 +209,7 @@ async def test_named_params_forwarded_into_advisor_executor_subcall(): them, e.g. Vertex AI rejecting ``clear_thinking_20251015`` context_management edits with: ``strategy requires thinking to be enabled or adaptive``. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) @@ -238,7 +238,7 @@ async def test_named_params_forwarded_into_advisor_executor_subcall(): return _text_resp("Final answer.") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_handler, ): await anthropic_messages( @@ -289,7 +289,7 @@ async def test_pre_request_hook_override_does_not_collide_with_explicit_kwargs() Regression for Greptile P2 on PR #27810. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages, ) @@ -326,11 +326,11 @@ async def test_pre_request_hook_override_does_not_collide_with_explicit_kwargs() with ( patch( - "litellm.llms.anthropic.pass_through.messages.handler._execute_pre_request_hooks", + "litellm.llms.anthropic.messages.native.handler._execute_pre_request_hooks", side_effect=fake_pre_request_hooks, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_handler, ), ): diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_agentic_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_agentic_streaming_iterator.py rename to tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py index 56fa3015d44..90a6ae3a574 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_agentic_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py @@ -11,7 +11,7 @@ import pytest from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES -from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( +from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, AgenticAnthropicStreamingIterator, _handle_content_block_delta, diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_effort.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_effort.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_effort.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_effort.py index 7885cc69b19..94476139313 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_effort.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_effort.py @@ -7,7 +7,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.llms.anthropic.common_utils import AnthropicError -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_encrypted_reasoning.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py similarity index 96% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_encrypted_reasoning.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py index ca81147da4c..cd6d2d9be96 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_encrypted_reasoning.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_encrypted_reasoning.py @@ -1,7 +1,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py similarity index 93% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index f4920505722..c11f58faec4 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -6,6 +6,7 @@ import uuid from typing import Any, Dict, Final, List import httpx +import responses import respx import pytest from fastapi.testclient import TestClient @@ -31,12 +32,63 @@ from litellm.utils import _invalidate_model_cost_lowercase_map import unittest.mock -def test_anthropic_experimental_pass_through_messages_handler(): +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", [False, True], ids=["complete", "stream"]) +async def test_openai_messages_forwards_stop_sequences_to_chat_completions( + monkeypatch: pytest.MonkeyPatch, is_async: bool, stream: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + response: Final = ( + httpx.Response( + 200, + json={ + "id": "chatcmpl-stop-sequences", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 0, "total_tokens": 1}, + }, + ) + if not stream + else httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text='data: {"id":"chatcmpl-stop-sequences","object":"chat.completion.chunk","created":1,' + '"model":"gpt-4o-mini","choices":[{"index":0,"delta":{"content":""},"finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n", + ) + ) + with respx.mock as upstream: + chat: Final = upstream.post("https://api.openai.com/v1/chat/completions").mock(return_value=response) + create: Final = litellm.anthropic.messages.acreate if is_async else litellm.anthropic.messages.create + pending: Final = create( + model="openai/gpt-4o-mini", + max_tokens=80, + messages=[{"role": "user", "content": "hi"}], + stop_sequences=["STOPPROBE", "ENDPROBE"], + stream=stream, + api_key="test-api-key", + ) + result: Final = await pending if is_async else pending + if stream: + chunks: Final = [chunk async for chunk in result] if is_async else list(result) + assert "message_stop" in "".join(chunk.decode() if isinstance(chunk, bytes) else chunk for chunk in chunks) + else: + assert result["content"] == [{"type": "text", "text": ""}] + body: Final = json.loads(chat.calls.last.request.content) + assert body["stop"] == ["STOPPROBE", "ENDPROBE"] + assert "stop_sequences" not in body + + +@pytest.mark.parametrize("stop_sequences", [None, []], ids=["no-stops", "empty-stops"]) +def test_anthropic_messages_handler(stop_sequences): """ Test that api key is passed to litellm.responses for OpenAI models. OpenAI and Azure models are routed directly to the Responses API. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -47,6 +99,7 @@ def test_anthropic_experimental_pass_through_messages_handler(): messages=[{"role": "user", "content": "Hello, how are you?"}], model="openai/claude-3-5-sonnet-20240620", api_key="test-api-key", + stop_sequences=stop_sequences, ) except (ValueError, TypeError, AttributeError) as e: print(f"Error: {e}") @@ -116,12 +169,12 @@ async def test_openai_model_does_not_forward_stream_options_to_responses_api(): assert "stream_options" not in request_body -def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): +def test_anthropic_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): """ Test that api key, api base, and extra kwargs are forwarded to litellm.completion for Azure models. Azure models are routed through chat/completions (not the Responses API). """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -148,7 +201,7 @@ async def test_anthropic_messages_sanitizes_empty_text_blocks_before_dispatch(): """Regression test for #22930. The unified /v1/messages path must strip empty text blocks before forwarding, otherwise Anthropic returns 400 "text content blocks must be non-empty".""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler msgs = [ { @@ -186,7 +239,7 @@ async def test_anthropic_messages_sanitizes_empty_text_blocks_before_dispatch(): @pytest.mark.asyncio async def test_anthropic_messages_sanitizes_tool_use_ids_before_dispatch(): - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler msgs = [ { @@ -230,14 +283,14 @@ async def _async_return(value): return value -def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): +def test_anthropic_messages_handler_custom_llm_provider(): """ Test that litellm.completion is called when a custom LLM provider is given. Provider resolution now happens exactly once, inside litellm.completion itself (BerriAI/litellm#37716), so the handler passes the original unresolved model through. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -321,7 +374,7 @@ def test_openai_model_with_thinking_converts_to_reasoning(): OpenAI models are routed directly to the Responses API, so we verify that litellm.responses() is called with `reasoning` properly set. """ - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -361,7 +414,7 @@ class TestThinkingParameterTransformation: def test_claude_model_preserves_thinking_with_budget_tokens(self): """Test that Claude models get thinking parameter passed through with exact budget_tokens.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -376,7 +429,7 @@ class TestThinkingParameterTransformation: def test_non_claude_model_converts_thinking_to_reasoning_effort(self): """Test that non-Claude models convert thinking to reasoning_effort.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -394,7 +447,7 @@ class TestThinkingParameterTransformation: def test_translate_thinking_for_model_summary_when_enabled(self): """When reasoning_auto_summary is True, summary='detailed' is injected.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -412,7 +465,7 @@ class TestThinkingParameterTransformation: def test_translate_thinking_for_model_preserves_user_summary(self): """User-provided summary is always preserved regardless of flag.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -429,7 +482,7 @@ class TestThinkingSummaryPreservation: def test_thinking_summary_concise_preserved_for_openai(self): """User-provided summary='concise' should not be replaced with 'detailed'.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -445,7 +498,7 @@ class TestThinkingSummaryPreservation: def test_thinking_summary_auto_preserved_for_openai(self): """User-provided summary='auto' should be preserved.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -462,7 +515,7 @@ class TestThinkingSummaryPreservation: def test_summary_added_when_auto_summary_enabled(self): """When reasoning_auto_summary is True, summary='detailed' is added.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -487,7 +540,7 @@ class TestThinkingSummaryPreservation: def test_no_summary_by_default_string_reasoning(self): """By default (reasoning_auto_summary=False), summary is not added for string reasoning_effort.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -510,7 +563,7 @@ class TestThinkingSummaryPreservation: def test_no_summary_by_default_dict_reasoning(self): """By default (reasoning_auto_summary=False), summary is not injected into dict reasoning_effort.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -533,7 +586,7 @@ class TestThinkingSummaryPreservation: def test_summary_added_when_env_var_set(self, monkeypatch): """When LITELLM_REASONING_AUTO_SUMMARY env var is true, summary is added.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -560,7 +613,7 @@ class TestThinkingSummaryPreservation: def test_user_provided_summary_preserved_even_when_flag_off(self): """When user already set summary in dict reasoning_effort, it's preserved regardless of flag.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) @@ -581,7 +634,7 @@ class TestThinkingSummaryPreservation: def test_openai_model_with_thinking_summary_end_to_end(self): """End-to-end: anthropic_messages_handler should preserve thinking.summary for OpenAI models.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -610,7 +663,7 @@ class TestThinkingSummaryPreservation: def test_responses_adapter_preserves_summary(self): """translate_thinking_to_reasoning should include summary when user provides it.""" - from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( + from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -621,7 +674,7 @@ class TestThinkingSummaryPreservation: def test_responses_adapter_no_summary_by_default(self): """translate_thinking_to_reasoning should not include summary by default (opt-in).""" import litellm - from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( + from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -637,7 +690,7 @@ class TestThinkingSummaryPreservation: def test_translate_thinking_for_model_preserves_summary(self): """translate_thinking_for_model should include summary in reasoning_effort dict when user provides it.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -651,7 +704,7 @@ class TestThinkingSummaryPreservation: def test_translate_thinking_for_model_disabled_stays_plain_string_when_auto_summary_enabled(self): """Disabled thinking must stay a plain string even when reasoning_auto_summary is on.""" import litellm - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -690,7 +743,7 @@ def _empty_block_msgs(): def test_handler_strips_when_no_presanitized_flag(): """Sync entry point (no async wrapper): handler must still sanitize.""" - from litellm.llms.anthropic.pass_through.messages import handler, utils + from litellm.llms.anthropic.messages.native import handler, utils with patch.object( utils, @@ -710,7 +763,7 @@ def test_handler_strips_when_no_presanitized_flag(): def test_handler_skips_strip_when_presanitized(): """Async wrapper already sanitized -> handler must NOT rescan.""" - from litellm.llms.anthropic.pass_through.messages import handler, utils + from litellm.llms.anthropic.messages.native import handler, utils with patch.object( utils, @@ -731,7 +784,7 @@ def test_handler_skips_strip_when_presanitized(): def test_handler_flattens_replayed_unencrypted_web_search_results(): """Synthesized search blocks replayed as history must reach the provider as text.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler captured = {} @@ -786,7 +839,7 @@ def test_handler_flattens_replayed_unencrypted_web_search_results(): def test_presanitized_flag_not_leaked_to_provider_params(): """The private sentinel must be popped, never forwarded as a request param.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler captured = {} @@ -815,7 +868,7 @@ def test_presanitized_flag_not_leaked_to_provider_params(): @pytest.mark.asyncio async def test_async_wrapper_sets_presanitized_and_sanitizes_once(): """End-to-end: wrapper sanitizes (once) AND signals the handler to skip.""" - from litellm.llms.anthropic.pass_through.messages import handler, utils + from litellm.llms.anthropic.messages.native import handler, utils captured = {} @@ -859,7 +912,7 @@ def _gate_stubs(monkeypatch): provider config handed to the native passthrough path and ``translation_calls`` counts hits on the Anthropic->OpenAI translation handlers. """ - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler captured = {} translation_calls = {"count": 0} @@ -889,7 +942,7 @@ def _gate_stubs(monkeypatch): def test_gate_passthrough_when_supported_endpoints_opts_in(monkeypatch): """provider=openai + model_info.supported_endpoints containing /v1/messages must route to the native passthrough config, NOT the translation handlers.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) from litellm.llms.openai_like.messages.transformation import ( @@ -915,7 +968,7 @@ def test_gate_passthrough_when_supported_endpoints_opts_in(monkeypatch): def test_gate_translates_when_supported_endpoints_absent(monkeypatch): """Default behavior is unchanged: without the /v1/messages opt-in, an openai deployment is translated (Responses API), never passed through natively.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -937,7 +990,7 @@ def test_gate_translates_when_supported_endpoints_absent(monkeypatch): def test_gate_passthrough_skipped_when_only_chat_completions_supported(monkeypatch): """A deployment that lists only /v1/chat/completions is still translated; the opt-in is specifically the /v1/messages entry.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -970,7 +1023,7 @@ def test_gate_passthrough_forwards_cache_control_ttl_only_when_deployment_opts_i ): """The passthrough config strips cache_control.ttl unless the deployment sets model_info.cache_control_ttl to exactly true.""" - from litellm.llms.anthropic.pass_through.messages.handler import ( + from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -1321,7 +1374,7 @@ class TestMessagesStreamingSuccessLogging: @pytest.mark.asyncio async def test_responses_bridge_streaming_emits_success_logging(self, capture_success_payloads): """The Responses bridge, which is the default for openai/ deployments.""" - from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( + from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, ) @@ -1362,14 +1415,14 @@ class TestMessagesStreamingSuccessLogging: """The chat-completions bridge, reached via litellm.use_chat_completions_url_for_anthropic_messages. Its router lookup is stubbed to what an SDK caller with no proxy running already resolves to.""" - from litellm.llms.anthropic.pass_through.adapters.handler import ( + from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) _bind_logging_worker_to_running_loop() with patch( - "litellm.llms.anthropic.pass_through.adapters.handler._proxy_router_fallback", + "litellm.llms.anthropic.messages.adapters.handler._proxy_router_fallback", return_value=None, ): sse_stream = await LiteLLMMessagesToCompletionTransformationHandler.async_anthropic_messages_handler( @@ -1422,7 +1475,7 @@ async def test_anthropic_messages_maps_provider_exception_before_failure_logging The 403 row pins the upstream status on the way through the mapper: Anthropic's documented permission_error must reach the caller as a 403, never as the mapper's APIConnectionError 500 fallthrough.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler capture = _FailureCapture() monkeypatch.setattr(litellm, "callbacks", [capture]) @@ -1464,7 +1517,7 @@ async def test_anthropic_messages_leaves_non_provider_failures_unmapped(): """The mapping boundary is for provider failures only. A request rejected before the provider call (here invalid metadata) must surface as the original exception, not as the mapper's APIConnectionError, whose message embeds a server traceback.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler def upstream_must_not_be_called(request: httpx.Request) -> httpx.Response: raise AssertionError("the provider must not be called for a request rejected locally") @@ -1510,7 +1563,7 @@ def _recording_client(seen_urls: list[str]) -> AsyncHTTPHandler: @pytest.mark.asyncio async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_default(monkeypatch): - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) monkeypatch.setenv("DEEPSEEK_ANTHROPIC_API_BASE", "https://deepseek.internal.example/anthropic") @@ -1530,7 +1583,7 @@ async def test_provider_messages_api_base_env_is_not_shadowed_by_the_chat_defaul @pytest.mark.asyncio async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthropic(): """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards = [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}] client_betas = "dangerous-tool-use-2026-09-03,interleaved-thinking-2025-05-14" @@ -1578,7 +1631,7 @@ async def test_anthropic_messages_forwards_safeguards_and_unknown_beta_to_anthro @pytest.mark.asyncio async def test_anthropic_messages_streaming_forwards_safeguards_and_keeps_safeguard_results(): """Shapes are what Claude Code 2.1.278 sends and api.anthropic.com returns, captured 2026-09-21.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards = [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}] tool_verdicts = {"toolu_01": {"type": "evaluated", "outcome": "not_flagged"}} @@ -1683,7 +1736,7 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet local_beta_headers_config, client_headers ): """Bedrock Invoke takes betas in the body's `anthropic_beta` and 400s on `safeguards` without the beta, so the beta rides along with the field.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards, safeguard_results = _claude_code_auto_mode_request() captured: dict[str, object] = {} @@ -1709,16 +1762,20 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet @pytest.mark.asyncio @pytest.mark.parametrize("client_headers", _CLIENT_BETA_HEADERS) async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_beta_to_vertex( - local_beta_headers_config, client_headers -): + local_beta_headers_config: None, client_headers: dict[str, str] +) -> None: """Vertex rawPredict takes the beta as the `anthropic-beta` header and 400s on `safeguards` without it, so the beta rides along with the field.""" - from litellm.llms.anthropic.pass_through.messages import handler - from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + from litellm.llms.anthropic.messages.native import handler safeguards, safeguard_results = _claude_code_auto_mode_request() captured: dict[str, object] = {} - with patch.object(VertexBase, "_ensure_access_token", return_value=("test-token", "test-project")): + with responses.RequestsMock(assert_all_requests_are_fired=False) as oauth: + oauth.add( + responses.POST, + "https://oauth2.googleapis.com/token", + json={"access_token": "test-token", "expires_in": 3600}, + ) response = await handler.anthropic_messages( max_tokens=16, messages=[{"role": "user", "content": "hi"}], @@ -1726,7 +1783,14 @@ async def test_anthropic_messages_forwards_safeguards_and_dangerous_tool_use_bet custom_llm_provider="vertex_ai", vertex_project="test-project", vertex_location="global", - vertex_credentials="{}", + vertex_credentials=json.dumps( + { + "type": "authorized_user", + "client_id": "synthetic-client", + "client_secret": "synthetic-secret", + "refresh_token": "synthetic-refresh", + } + ), client=_upstream_answering_with(safeguard_results, captured), safeguards=safeguards, extra_headers=client_headers, @@ -1820,7 +1884,7 @@ def _native_upstream(*, stream: bool, model: str, **extra: object) -> AsyncHTTPH async def _native_messages(model: str, upstream: AsyncHTTPHandler, *, stream: bool, **extra: object) -> object: - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler safeguards, _ = _claude_code_auto_mode_request() with patch("litellm.proxy.proxy_server.general_settings", {"safeguards_classifier_model": "classifier"}): @@ -2268,7 +2332,7 @@ async def test_anthropic_messages_bedrock_dynamic_region(): async def test_anthropic_messages_with_safeguards_and_no_classifier_keep_the_upstream_400_for_malformed_messages( malformed_messages: object, ) -> None: - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler def upstream_rejects_the_messages(request: httpx.Request) -> httpx.Response: return httpx.Response( @@ -2298,7 +2362,7 @@ async def test_anthropic_messages_with_safeguards_and_no_classifier_keep_the_ups def test_native_messages_tool_config_point_marks_last_tool_and_drops_the_key(monkeypatch): """LIT-9880 on /v1/messages: the hook wrote the tool_config point back into kwargs and the native path dropped it silently, so the tools went out without any cache breakpoint.""" - from litellm.llms.anthropic.pass_through.messages import handler + from litellm.llms.anthropic.messages.native import handler captured = {} diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_per_turn_control.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_per_turn_control.py index 05c12c6a285..e29f962ec8c 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_per_turn_control.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_per_turn_control.py @@ -2,7 +2,7 @@ import pytest from litellm import anthropic_beta_headers_manager from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.openai_like.json_loader import SimpleProviderConfig diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_service_tier.py similarity index 92% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_service_tier.py index 6472ccd1b2e..75d5b583cc8 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_service_tier.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_service_tier.py @@ -2,7 +2,7 @@ from typing import Final import pytest -from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils +from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils REQUEST: Final[dict[str, object]] = { "max_tokens": 64, diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_speed.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_speed.py similarity index 96% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_speed.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_speed.py index 45c6ebaa39b..9be3cdf55b3 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_speed.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_speed.py @@ -1,9 +1,9 @@ import litellm import pytest -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( AnthropicMessagesRequestUtils, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_structured_outputs.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_structured_outputs.py index db3ff58d80a..5d9ad409f15 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_structured_outputs.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_structured_outputs.py @@ -8,7 +8,7 @@ from typing import Final import pytest import litellm -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.types.router import GenericLiteLLMParams diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py rename to tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 457d5d5e1c8..aa6625561ba 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -10,7 +10,7 @@ from pydantic import JsonValue import litellm from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.anthropic.wif import get_anthropic_wif_token diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_content_after_stop_reason.py b/tests/unit/llms/anthropic/messages/native/test_content_after_stop_reason.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_content_after_stop_reason.py rename to tests/unit/llms/anthropic/messages/native/test_content_after_stop_reason.py index f57a872c82a..1accce8383b 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_content_after_stop_reason.py +++ b/tests/unit/llms/anthropic/messages/native/test_content_after_stop_reason.py @@ -17,7 +17,7 @@ from typing import List import pytest -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_fake_stream_iterator.py b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/messages/test_fake_stream_iterator.py rename to tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py index 476cea2833c..233145f2706 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_fake_stream_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py @@ -2,7 +2,7 @@ import json from collections.abc import Iterable, Mapping from typing import Final -from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( +from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_mcp_handler.py b/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py similarity index 95% rename from tests/unit/llms/anthropic/pass_through/messages/test_mcp_handler.py rename to tests/unit/llms/anthropic/messages/native/test_mcp_handler.py index 3aca6181d12..73c2edbce19 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_mcp_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_mcp_handler.py @@ -4,13 +4,13 @@ from unittest.mock import AsyncMock, patch import pytest -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) -from litellm.llms.anthropic.pass_through.messages.handler import ( +from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) -from litellm.llms.anthropic.pass_through.messages.mcp_handler import ( +from litellm.llms.anthropic.messages.native.mcp_handler import ( _build_tool_result_message, _extract_tool_use_blocks, ) @@ -39,7 +39,7 @@ def test_anthropic_messages_handler_routes_litellm_proxy_mcp_to_the_gateway(): dispatch makes the whole feature unreachable while every unit test still passes. """ with patch( - "litellm.llms.anthropic.pass_through.messages.mcp_handler.anthropic_messages_with_mcp", + "litellm.llms.anthropic.messages.native.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: result = anthropic_messages_handler( @@ -59,7 +59,7 @@ def test_anthropic_messages_handler_routes_litellm_proxy_mcp_to_the_gateway(): def test_anthropic_messages_handler_skips_the_gateway_on_recursion(): """The gateway's own follow-up call must not re-enter the gateway.""" with patch( - "litellm.llms.anthropic.pass_through.messages.mcp_handler.anthropic_messages_with_mcp", + "litellm.llms.anthropic.messages.native.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: with pytest.raises(ValueError, match="anthropic_messages_handler is not implemented for sync calls"): @@ -78,7 +78,7 @@ def test_anthropic_messages_handler_skips_the_gateway_on_recursion(): def test_anthropic_messages_handler_leaves_native_tools_alone(): """A plain Anthropic tool is not an MCP reference and must not reach the gateway.""" with patch( - "litellm.llms.anthropic.pass_through.messages.mcp_handler.anthropic_messages_with_mcp", + "litellm.llms.anthropic.messages.native.mcp_handler.anthropic_messages_with_mcp", new=AsyncMock(return_value={"routed": True}), ) as routed: with pytest.raises(ValueError, match="anthropic_messages_handler is not implemented for sync calls"): @@ -161,7 +161,7 @@ async def test_anthropic_messages_with_mcp_forwards_the_callers_mcp_credentials( token, per-user env) silently returns nothing while the model claims it has no access. Only a no-auth server would look healthy. """ - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext context = MCPRequestContext( @@ -246,7 +246,7 @@ async def test_anthropic_messages_with_mcp_hands_execution_the_requests_served_t """ from mcp.types import Tool - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext served = [ @@ -311,7 +311,7 @@ async def test_anthropic_messages_with_mcp_stops_when_every_tool_call_is_skipped Anthropic rejects that, so the caller would get an unhandled 400 from the middle of the loop rather than the model's own answer. """ - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext tool_use_response = { @@ -349,7 +349,7 @@ async def test_anthropic_messages_with_mcp_stops_when_every_tool_call_is_skipped @pytest.mark.asyncio async def test_anthropic_messages_with_mcp_classifies_only_the_response_the_client_gets(): - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext from litellm.router import Router @@ -409,7 +409,7 @@ async def _mcp_safeguards_turn_recording_beta_map_loads(custom_llm_provider: str import threading from litellm import anthropic_beta_headers_manager - from litellm.llms.anthropic.pass_through.messages import mcp_handler + from litellm.llms.anthropic.messages.native import mcp_handler from litellm.responses.mcp.request_context import MCPRequestContext loader_threads: Final[list[int]] = [] # mutable-ok: records the thread each beta map load ran on diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_mid_conversation_system.py b/tests/unit/llms/anthropic/messages/native/test_mid_conversation_system.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/messages/test_mid_conversation_system.py rename to tests/unit/llms/anthropic/messages/native/test_mid_conversation_system.py index f527b4912d5..a1615c01d08 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_mid_conversation_system.py +++ b/tests/unit/llms/anthropic/messages/native/test_mid_conversation_system.py @@ -1,6 +1,6 @@ from collections import Counter -from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( +from litellm.llms.anthropic.messages.native.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, convert_mid_conversation_system_turns, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_parallel_tool_calls.py b/tests/unit/llms/anthropic/messages/native/test_parallel_tool_calls.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_parallel_tool_calls.py rename to tests/unit/llms/anthropic/messages/native/test_parallel_tool_calls.py index 229eff4275a..5b2e285fed7 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_parallel_tool_calls.py +++ b/tests/unit/llms/anthropic/messages/native/test_parallel_tool_calls.py @@ -2,7 +2,7 @@ from typing import List -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import ( diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_auto_summary_messages.py b/tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/messages/test_reasoning_auto_summary_messages.py rename to tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py index cc951421b10..7334b4eeaa1 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_auto_summary_messages.py +++ b/tests/unit/llms/anthropic/messages/native/test_reasoning_auto_summary_messages.py @@ -14,7 +14,7 @@ from unittest.mock import MagicMock, patch import litellm -from litellm.llms.anthropic.pass_through.messages.handler import ( +from litellm.llms.anthropic.messages.native.handler import ( anthropic_messages_handler, ) @@ -30,10 +30,10 @@ def _call_handler_and_capture_optional_params(thinking=None, **extra_kwargs): captured = {} with patch( - "litellm.llms.anthropic.pass_through.messages.handler." + "litellm.llms.anthropic.messages.native.handler." "base_llm_http_handler" ) as mock_handler, patch( - "litellm.llms.anthropic.pass_through.messages.handler." + "litellm.llms.anthropic.messages.native.handler." "ProviderConfigManager" ) as mock_pcm: # Make get_provider_anthropic_messages_config return a non-None config diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py b/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py rename to tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py index 284c6ea532d..50dc0deabfd 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_reasoning_effort_translation.py +++ b/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py @@ -8,7 +8,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.llms.anthropic.common_utils import AnthropicError -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import ( diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py rename to tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py index 7a4f3aa5f90..d525ccfec37 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_request_optional_param_utils.py +++ b/tests/unit/llms/anthropic/messages/native/test_request_optional_param_utils.py @@ -9,7 +9,7 @@ Regression tests for the /v1/messages request-parse fast paths: import pytest import litellm -from litellm.llms.anthropic.pass_through.messages.utils import ( +from litellm.llms.anthropic.messages.native.utils import ( AnthropicMessagesRequestUtils, anthropic_messages_optional_param_keys, ) diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py b/tests/unit/llms/anthropic/messages/native/test_response_cache.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py rename to tests/unit/llms/anthropic/messages/native/test_response_cache.py index 1df4ccec02c..133bcd96433 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py +++ b/tests/unit/llms/anthropic/messages/native/test_response_cache.py @@ -13,8 +13,8 @@ from litellm.caching.caching_handler import LLMCachingHandler from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.messages import handler -from litellm.llms.anthropic.pass_through.messages.response_cache import ( +from litellm.llms.anthropic.messages.native import handler +from litellm.llms.anthropic.messages.native.response_cache import ( AnthropicMessagesStreamCacheWriter, CachedAnthropicMessagesStreamIterator, ) @@ -275,7 +275,7 @@ async def test_abandoned_stream_is_not_cached(local_cache, request_kwargs, monke async def test_cached_stream_replay_logs_once_when_polled_after_exhaustion(): from unittest.mock import AsyncMock, MagicMock, patch - from litellm.llms.anthropic.pass_through.messages.response_cache import ( + from litellm.llms.anthropic.messages.native.response_cache import ( CachedAnthropicMessagesStreamIterator, ) from litellm.proxy.pass_through_endpoints.streaming_handler import ( diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_sse_wrapper.py b/tests/unit/llms/anthropic/messages/native/test_sse_wrapper.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/messages/test_sse_wrapper.py rename to tests/unit/llms/anthropic/messages/native/test_sse_wrapper.py index d22478e16ef..cd6b3fa006d 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_sse_wrapper.py +++ b/tests/unit/llms/anthropic/messages/native/test_sse_wrapper.py @@ -2,7 +2,7 @@ import pytest from fastapi.testclient import TestClient -from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, ) from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py rename to tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py index 4ed9054f3d3..dd50f60250a 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_streaming_iterator.py @@ -11,8 +11,8 @@ import pytest from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.messages import streaming_iterator as streaming_iterator_module -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( +from litellm.llms.anthropic.messages.native import streaming_iterator as streaming_iterator_module +from litellm.llms.anthropic.messages.native.streaming_iterator import ( INCOMPLETE_STREAM_ERROR_MESSAGE, AnthropicMessagesStreamHiddenParams, AnthropicMessagesStreamingResponse, @@ -26,8 +26,8 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( is_provider_error_chunk, parse_anthropic_error_event, ) -from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig -from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator +from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig +from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig from litellm.types.utils import Choices, Message, ModelResponse @@ -1157,7 +1157,7 @@ async def test_abort_upstream_logs_warning_when_aclose_raises(caplog): async def test_enqueue_for_client_returns_false_when_already_detached(): """_enqueue_for_client must return False immediately (without touching the queue) when client_detached is already set before the call.""" - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) @@ -1174,7 +1174,7 @@ async def test_enqueue_for_client_returns_false_when_already_detached(): async def test_enqueue_for_client_returns_false_when_client_detaches_while_queue_full(): """_enqueue_for_client must return False (and cancel the put) when the queue is full and client_detached fires before space becomes available.""" - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/__init__.py b/tests/unit/llms/anthropic/messages/responses_adapters/__init__.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/responses_adapters/__init__.py rename to tests/unit/llms/anthropic/messages/responses_adapters/__init__.py diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py rename to tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 8fe6bd31082..7de722e1171 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -16,7 +16,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, ) -from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( +from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, ) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py rename to tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py index e1dded214bf..5af8e69a0ea 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -1,6 +1,6 @@ """ Tests for AnthropicResponsesStreamWrapper -(litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py) +(litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py) """ import asyncio @@ -18,8 +18,8 @@ from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) -from litellm.llms.anthropic.pass_through.messages.utils import INCOMPLETE_STREAM_ERROR_MESSAGE -from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import ( +from litellm.llms.anthropic.messages.native.utils import INCOMPLETE_STREAM_ERROR_MESSAGE +from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) from litellm.types.llms.openai import ResponseFailedEvent, ResponsesAPIResponse diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py rename to tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index 829000c0c5f..f6fcb9e6f1e 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -1,6 +1,6 @@ """ Tests for LiteLLMAnthropicToResponsesAPIAdapter -(litellm/llms/anthropic/pass_through/responses_adapters/transformation.py) +(litellm/llms/anthropic/messages/responses_adapters/transformation.py) """ import json @@ -22,7 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, encrypted_reasoning_signature, ) -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig @@ -2413,7 +2413,7 @@ class TestPromptCacheBreakpointToResponses: assert log_output == self.LOG_OUTPUT def test_prompt_cache_options_forwarded_to_responses_kwargs(self): - from litellm.llms.anthropic.pass_through.responses_adapters.handler import ( + from litellm.llms.anthropic.messages.responses_adapters.handler import ( _build_responses_kwargs, ) diff --git a/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py b/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py index 5b0f743b836..810dfffaf8b 100644 --- a/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py +++ b/tests/unit/llms/anthropic/messages/test_advisor_orchestration.py @@ -71,7 +71,7 @@ def _make_advisor_tool_use_response( def test_can_handle_edge_cases(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -99,7 +99,7 @@ async def test_anthropic_native_interceptor_skipped(): For provider=anthropic, can_handle() must return False. The interceptor must never call handle(). """ - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -115,7 +115,7 @@ async def test_anthropic_native_interceptor_skipped(): @pytest.mark.asyncio async def test_loop_no_advisor_call(): """Executor returns text on first try — no advisor call, loop exits immediately.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, _call_messages_handler, ) @@ -124,7 +124,7 @@ async def test_loop_no_advisor_call(): executor_response = _make_text_response(final_text) with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", new_callable=AsyncMock, return_value=executor_response, ) as mock_call: @@ -157,7 +157,7 @@ async def test_loop_one_advisor_call(): Executor calls advisor once → advisor responds → executor produces final text. Total calls: 3 (executor, advisor, executor-final). """ - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -185,7 +185,7 @@ async def test_loop_one_advisor_call(): return final_resp # executor: final answer with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -217,7 +217,7 @@ async def test_loop_one_advisor_call(): @pytest.mark.asyncio async def test_loop_max_uses_raises(): """Loop exceeding max_uses must raise AdvisorMaxIterationsError.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, AdvisorOrchestrationHandler, ) @@ -238,7 +238,7 @@ async def test_loop_max_uses_raises(): return advisor_tool_use_resp with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -261,17 +261,17 @@ async def test_loop_max_uses_raises(): @pytest.mark.asyncio async def test_loop_streaming_wraps_response(): """stream=True: final response must be wrapped in FakeAnthropicMessagesStreamIterator.""" - from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) executor_response = _make_text_response("Hello, world!") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", new_callable=AsyncMock, return_value=executor_response, ): @@ -307,7 +307,7 @@ async def test_prior_advisor_blocks_replaced_in_history(): History containing server_tool_use + advisor_tool_result blocks gets collapsed to text before forwarding to the executor. """ - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -340,7 +340,7 @@ async def test_prior_advisor_blocks_replaced_in_history(): return _make_text_response("Here is the efficient version.") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -380,7 +380,7 @@ async def test_advisor_tool_translated_for_executor(): """ The executor must receive a regular tool definition (not advisor_20260301 type). """ - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -392,7 +392,7 @@ async def test_advisor_tool_translated_for_executor(): return _make_text_response("Done.") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -422,7 +422,7 @@ async def test_advisor_tool_translated_for_executor(): @pytest.mark.asyncio async def test_max_uses_zero_raises_on_first_advisor_call(): """max_uses=0 must cause AdvisorMaxIterationsError on the first advisor call.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, AdvisorOrchestrationHandler, ) @@ -434,7 +434,7 @@ async def test_max_uses_zero_raises_on_first_advisor_call(): return advisor_tool_use_resp # executor always tries to call advisor with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -457,7 +457,7 @@ async def test_max_uses_zero_raises_on_first_advisor_call(): @pytest.mark.asyncio async def test_missing_advisor_model_raises_value_error(): """handle() must raise ValueError when the advisor tool has no model field.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -484,7 +484,7 @@ async def test_missing_advisor_model_raises_value_error(): async def test_max_uses_none_falls_back_to_default(): """When max_uses is absent, the handler uses ADVISOR_MAX_USES from constants.""" import litellm.constants as _c - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, AdvisorOrchestrationHandler, ) @@ -498,7 +498,7 @@ async def test_max_uses_none_falls_back_to_default(): return advisor_tool_use_resp with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -532,7 +532,7 @@ ADVISOR_TOOL_WITH_CREDS = { async def _run_advisor_and_capture_subcall_kwargs(): """Run one advisor turn and return the kwargs of the advisor sub-call.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -557,11 +557,11 @@ async def _run_advisor_and_capture_subcall_kwargs(): with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url", + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url", ), ): h = AdvisorOrchestrationHandler() @@ -581,7 +581,7 @@ async def test_advisor_creds_dropped_when_proxy_opt_in_disabled(): """On the proxy without opt-in, the caller's advisor api_base/api_key must NOT reach the sub-call (would redirect it / leak the server key).""" with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=False, ): captured = await _run_advisor_and_capture_subcall_kwargs() @@ -593,7 +593,7 @@ async def test_advisor_creds_dropped_when_proxy_opt_in_disabled(): async def test_advisor_creds_honored_when_proxy_opt_in_enabled(): """With the admin opt-in, the documented clientside routing still works.""" with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): captured = await _run_advisor_and_capture_subcall_kwargs() @@ -626,7 +626,7 @@ def test_allow_client_side_advisor_credentials_reads_proxy_flag(): """The gate mirrors the proxy's allow_client_side_credentials opt-in.""" import sys - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _allow_client_side_advisor_credentials, ) @@ -650,7 +650,7 @@ def test_allow_client_side_advisor_credentials_defaults_true_outside_proxy(): import builtins import sys - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _allow_client_side_advisor_credentials, ) @@ -674,7 +674,7 @@ def test_advisor_gate_propagates_non_import_errors(): returning True.""" import sys - from litellm.llms.anthropic.pass_through.messages.interceptors import ( + from litellm.llms.anthropic.messages.native.interceptors import ( advisor, ) @@ -733,12 +733,12 @@ async def test_advisor_uses_tool_credentials_when_clientside_enabled(): def test_resolve_advisor_credentials_returns_none_when_gate_closed(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=False, ): result = _resolve_advisor_credentials(ADVISOR_TOOL_WITH_CREDS) @@ -746,18 +746,18 @@ def test_resolve_advisor_credentials_returns_none_when_gate_closed(): def test_resolve_advisor_credentials_allows_api_key_without_api_base(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_key": "sk-other"} with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url", + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url", side_effect=AssertionError("validate_url must not run without an api_base"), ), ): @@ -766,13 +766,13 @@ def test_resolve_advisor_credentials_allows_api_key_without_api_base(): def test_resolve_advisor_credentials_rejects_api_base_without_api_key(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_base": "https://other.example"} with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): with pytest.raises(ValueError, match="api_base"): @@ -781,7 +781,7 @@ def test_resolve_advisor_credentials_rejects_api_base_without_api_key(): @pytest.mark.parametrize("api_base", [None, "https://other.example"]) def test_resolve_advisor_credentials_rejects_an_oidc_secret_reference_api_key(api_base): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) @@ -793,17 +793,17 @@ def test_resolve_advisor_credentials_rejects_an_oidc_secret_reference_api_key(ap def test_resolve_advisor_credentials_validates_api_base_before_use(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url" + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url" ) as mock_validate, ): result = _resolve_advisor_credentials(ADVISOR_TOOL_WITH_CREDS) @@ -813,17 +813,17 @@ def test_resolve_advisor_credentials_validates_api_base_before_use(): def test_resolve_advisor_credentials_propagates_ssrf_error(): from litellm.litellm_core_utils.url_utils import SSRFError - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url", + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url", side_effect=SSRFError("URL targets a blocked address"), ), ): @@ -834,18 +834,18 @@ def test_resolve_advisor_credentials_propagates_ssrf_error(): def test_resolve_advisor_credentials_skips_validation_when_url_validation_disabled(): import litellm - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch.object(litellm, "user_url_validation", False), patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor.validate_url", + "litellm.llms.anthropic.messages.native.interceptors.advisor.validate_url", side_effect=AssertionError("validate_url must not run when user_url_validation is disabled"), ), ): @@ -857,7 +857,7 @@ def test_resolve_advisor_credentials_blocks_real_cloud_metadata_address(): """End-to-end (no mocked validate_url): a caller can't redirect the advisor sub-call to the cloud-metadata address even with an api_key.""" from litellm.litellm_core_utils.url_utils import SSRFError - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) @@ -867,7 +867,7 @@ def test_resolve_advisor_credentials_blocks_real_cloud_metadata_address(): "api_base": "https://169.254.169.254/latest/meta-data/", } with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): with pytest.raises(SSRFError): @@ -875,13 +875,13 @@ def test_resolve_advisor_credentials_blocks_real_cloud_metadata_address(): def test_resolve_advisor_credentials_rejects_non_https_api_base(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_key": "sk-other", "api_base": "http://8.8.8.8"} with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): with pytest.raises(ValueError, match="https"): @@ -891,14 +891,14 @@ def test_resolve_advisor_credentials_rejects_non_https_api_base(): def test_resolve_advisor_credentials_rejects_api_base_when_ssl_verify_disabled(): import litellm - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_key": "sk-other", "api_base": "https://8.8.8.8"} with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ), patch.object(litellm, "ssl_verify", False), @@ -910,13 +910,13 @@ def test_resolve_advisor_credentials_rejects_api_base_when_ssl_verify_disabled() def test_resolve_advisor_credentials_allows_real_public_ip_address(): """End-to-end (no mocked validate_url): a globally-routable literal IP api_base is honored when paired with an api_key.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( _resolve_advisor_credentials, ) tool = {**ADVISOR_TOOL, "api_key": "sk-other", "api_base": "https://8.8.8.8"} with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._allow_client_side_advisor_credentials", + "litellm.llms.anthropic.messages.native.interceptors.advisor._allow_client_side_advisor_credentials", return_value=True, ): result = _resolve_advisor_credentials(tool) @@ -935,7 +935,7 @@ async def test_advisor_sub_call_failure_is_tagged(): """When the advisor sub-call raises, the exception that propagates out of handle() must be tagged as an advisor orchestration failure.""" import litellm - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) from litellm.router_utils.cooldown_handlers import is_advisor_orchestration_failure @@ -954,7 +954,7 @@ async def test_advisor_sub_call_failure_is_tagged(): ) with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -977,7 +977,7 @@ async def test_advisor_max_iterations_failure_is_tagged(): """When the orchestration loop exceeds max_uses (the executor keeps calling the advisor), the AdvisorMaxIterationsError must be tagged so the healthy executor deployment is not cooled down.""" - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorMaxIterationsError, AdvisorOrchestrationHandler, ) @@ -993,7 +993,7 @@ async def test_advisor_max_iterations_failure_is_tagged(): return _make_advisor_tool_use_response() with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -1015,7 +1015,7 @@ async def test_executor_failure_is_not_tagged(): """A failure of the executor call (not advisor orchestration) must NOT be tagged — the selected deployment genuinely failed and should cool down.""" import litellm - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) from litellm.router_utils.cooldown_handlers import is_advisor_orchestration_failure @@ -1028,7 +1028,7 @@ async def test_executor_failure_is_not_tagged(): ) with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() @@ -1084,7 +1084,7 @@ def _router_with_advisor_deployment( @pytest.mark.asyncio async def test_advisor_sub_call_routes_through_proxy_router(): import litellm.proxy.proxy_server as proxy_server - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1107,7 +1107,7 @@ async def test_advisor_sub_call_routes_through_proxy_router(): with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch.object(proxy_server, "llm_router", router), @@ -1145,7 +1145,7 @@ async def test_advisor_sub_call_routes_through_proxy_router(): async def test_advisor_sub_call_routes_through_router_for_alias_and_wildcard(router_kwargs, advisor_model): """Alias and wildcard advisor models resolve through the router like exact model_list matches.""" import litellm.proxy.proxy_server as proxy_server - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1168,7 +1168,7 @@ async def test_advisor_sub_call_routes_through_router_for_alias_and_wildcard(rou with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch.object(proxy_server, "llm_router", router), @@ -1194,7 +1194,7 @@ async def test_advisor_sub_call_routes_through_router_for_alias_and_wildcard(rou async def test_advisor_sub_call_bypasses_router_for_unconfigured_model(): """An advisor model the router doesn't know about keeps the SDK-level path.""" import litellm.proxy.proxy_server as proxy_server - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1219,7 +1219,7 @@ async def test_advisor_sub_call_bypasses_router_for_unconfigured_model(): with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch.object(proxy_server, "llm_router", router), @@ -1243,7 +1243,7 @@ async def test_advisor_sub_call_client_override_bypasses_router(): """A caller-supplied api_key/api_base override must not be re-routed.""" import litellm import litellm.proxy.proxy_server as proxy_server - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1274,7 +1274,7 @@ async def test_advisor_sub_call_client_override_bypasses_router(): with ( patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ), patch.object(proxy_server, "llm_router", router), @@ -1309,7 +1309,7 @@ async def test_advisor_sub_call_client_override_bypasses_router(): @pytest.mark.asyncio async def test_advisor_context_excludes_in_sequence_system_rows(): - from litellm.llms.anthropic.pass_through.messages.interceptors.advisor import ( + from litellm.llms.anthropic.messages.native.interceptors.advisor import ( AdvisorOrchestrationHandler, ) @@ -1329,7 +1329,7 @@ async def test_advisor_context_excludes_in_sequence_system_rows(): return _make_text_response("Final answer.") with patch( - "litellm.llms.anthropic.pass_through.messages.interceptors.advisor._call_messages_handler", + "litellm.llms.anthropic.messages.native.interceptors.advisor._call_messages_handler", side_effect=mock_call, ): h = AdvisorOrchestrationHandler() diff --git a/tests/unit/llms/anthropic/pass_through/test_anthropic_native_passthrough_spend_logging.py b/tests/unit/llms/anthropic/messages/test_anthropic_native_passthrough_spend_logging.py similarity index 100% rename from tests/unit/llms/anthropic/pass_through/test_anthropic_native_passthrough_spend_logging.py rename to tests/unit/llms/anthropic/messages/test_anthropic_native_passthrough_spend_logging.py diff --git a/tests/unit/llms/anthropic/pass_through/test_caller_model_checks.py b/tests/unit/llms/anthropic/messages/test_caller_model_checks.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/test_caller_model_checks.py rename to tests/unit/llms/anthropic/messages/test_caller_model_checks.py index 5a45221a3ca..751f7cc3934 100644 --- a/tests/unit/llms/anthropic/pass_through/test_caller_model_checks.py +++ b/tests/unit/llms/anthropic/messages/test_caller_model_checks.py @@ -7,7 +7,7 @@ from starlette.requests import Request import litellm from litellm.caching.dual_cache import DualCache -from litellm.llms.anthropic.pass_through.caller_model_checks import ( +from litellm.llms.anthropic.messages.caller_model_checks import ( _caller_within_team_member_model_budget, caller_model_verdict, rate_limit_verdict, @@ -32,7 +32,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( from litellm.proxy.utils import InternalUsageCache, hash_token from litellm.router import Router from litellm.types.agents import AgentResponse -from tests.unit.llms.anthropic.pass_through.conftest import ProxyAuthState +from tests.unit.llms.anthropic.messages.conftest import ProxyAuthState CLASSIFIER: Final = "classifier" CLASSIFIER_BUDGET: Final = {CLASSIFIER: {"max_budget": 0.1, "budget_duration": "1d"}} diff --git a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py b/tests/unit/llms/anthropic/messages/test_convertible_blocks.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py rename to tests/unit/llms/anthropic/messages/test_convertible_blocks.py index eac75486da4..5fb6cc32dfb 100644 --- a/tests/unit/llms/anthropic/pass_through/test_convertible_blocks.py +++ b/tests/unit/llms/anthropic/messages/test_convertible_blocks.py @@ -6,13 +6,13 @@ from typing import Final import pytest from pydantic import JsonValue, TypeAdapter -from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter -from litellm.llms.anthropic.pass_through.convertible_blocks import ( +from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.llms.anthropic.messages.convertible_blocks import ( malformed_tool_result_text_error, unconvertible_anthropic_blocks, unconvertible_blocks_error, ) -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.vertex_ai.count_tokens.context import gemini_count_contents diff --git a/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py similarity index 97% rename from tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py rename to tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index e9450d025a6..3432c9503de 100644 --- a/tests/unit/llms/anthropic/pass_through/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -14,7 +14,7 @@ from typing import Any, Dict import pytest import litellm -from litellm.llms.anthropic.pass_through.utils import ( +from litellm.llms.anthropic.messages.utils import ( normalize_reasoning_effort_value, ) from litellm.router_utils.reasoning_effort_capability import ( @@ -156,7 +156,7 @@ class TestAdapterAdaptiveThinking: def test_messages_adapter_adaptive_returns_medium_default(self): """Adaptive thinking returns 'medium' as default reasoning_effort.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -168,7 +168,7 @@ class TestAdapterAdaptiveThinking: def test_messages_adapter_adaptive_overridden_by_output_config(self): """For adaptive thinking, output_config.effort overrides reasoning_effort.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) from litellm.types.llms.anthropic import AnthropicMessagesRequest @@ -191,7 +191,7 @@ class TestAdapterAdaptiveThinking: def test_responses_adapter_adaptive_with_output_config(self): """Responses adapter: adaptive thinking + output_config.effort.""" - from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( + from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) @@ -204,7 +204,7 @@ class TestAdapterAdaptiveThinking: def test_responses_adapter_adaptive_default_medium(self): """Responses adapter: adaptive thinking without output_config defaults to medium.""" - from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( + from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) diff --git a/tests/unit/llms/anthropic/pass_through/test_safeguards.py b/tests/unit/llms/anthropic/messages/test_safeguards.py similarity index 99% rename from tests/unit/llms/anthropic/pass_through/test_safeguards.py rename to tests/unit/llms/anthropic/messages/test_safeguards.py index 8ea9b693139..0537d0fbd38 100644 --- a/tests/unit/llms/anthropic/pass_through/test_safeguards.py +++ b/tests/unit/llms/anthropic/messages/test_safeguards.py @@ -12,11 +12,11 @@ import litellm from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.caller_model_checks import ( +from litellm.llms.anthropic.messages.caller_model_checks import ( CallerModelVerdict, _caller_within_team_member_model_budget, ) -from litellm.llms.anthropic.pass_through.safeguards import ( +from litellm.llms.anthropic.messages.safeguards import ( CLASSIFIER_MAX_TOKENS, CLASSIFIER_SYSTEM_PROMPT, TOOL_RESULT_CHAR_LIMIT, @@ -32,7 +32,7 @@ from litellm.llms.anthropic.pass_through.safeguards import ( requested_dangerous_tool_use, with_safeguard_results, ) -from litellm.llms.anthropic.pass_through.utils import proxy_sub_call_metadata +from litellm.llms.anthropic.messages.utils import proxy_sub_call_metadata from litellm.proxy._types import LiteLLM_EndUserTable, Litellm_EntityType, UserAPIKeyAuth from litellm.proxy.hooks.model_max_budget_limiter import ( PROXY_VirtualKeyModelMaxBudgetLimiter, @@ -44,7 +44,7 @@ from litellm.router import Router from litellm.types.llms.anthropic import AnthropicResponseContentBlockToolUse from litellm.types.router import RouterErrors from litellm.types.utils import Choices, Message, ModelResponse -from tests.unit.llms.anthropic.pass_through.conftest import ProxyAuthState +from tests.unit.llms.anthropic.messages.conftest import ProxyAuthState DANGEROUS_TOOL_USE_REQUEST: Final = [{"type": "dangerous_tool_use", "classifier_context": {"permission_mode": "auto"}}] LS: Final = ToolUseUnderReview(id="call_ls", name="Bash", input={"command": "ls"}) diff --git a/tests/unit/llms/anthropic/pass_through/test_stream_assembly.py b/tests/unit/llms/anthropic/messages/test_stream_assembly.py similarity index 98% rename from tests/unit/llms/anthropic/pass_through/test_stream_assembly.py rename to tests/unit/llms/anthropic/messages/test_stream_assembly.py index dbd49064e4b..f7d6b8d4f8c 100644 --- a/tests/unit/llms/anthropic/pass_through/test_stream_assembly.py +++ b/tests/unit/llms/anthropic/messages/test_stream_assembly.py @@ -3,7 +3,7 @@ from typing import Final import pytest -from litellm.llms.anthropic.pass_through.stream_assembly import ( +from litellm.llms.anthropic.messages.stream_assembly import ( anthropic_message_delta_fields, assemble_anthropic_sse_stream, is_anthropic_sse_stream, diff --git a/tests/unit/llms/anthropic/pass_through/test_pass_through_shim.py b/tests/unit/llms/anthropic/pass_through/test_pass_through_shim.py new file mode 100644 index 00000000000..4016e5301db --- /dev/null +++ b/tests/unit/llms/anthropic/pass_through/test_pass_through_shim.py @@ -0,0 +1,60 @@ +import importlib +import sys +from typing import Final + +import pytest + +OLD_PACKAGE: Final = "litellm.llms.anthropic.pass_through" +NEW_PACKAGE: Final = "litellm.llms.anthropic.messages" + + +def _forget_old_package(monkeypatch: pytest.MonkeyPatch) -> None: + for name in tuple(sys.modules): + if name == OLD_PACKAGE or name.startswith(OLD_PACKAGE + "."): + monkeypatch.delitem(sys.modules, name) + + +def test_old_deep_import_is_the_new_module_object_and_warns(monkeypatch: pytest.MonkeyPatch) -> None: + _forget_old_package(monkeypatch) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + old_handler: Final = importlib.import_module(f"{OLD_PACKAGE}.messages.handler") + from litellm.llms.anthropic.messages.native import handler as new_handler + + assert old_handler is new_handler + assert sys.modules[f"{OLD_PACKAGE}.messages.handler"] is new_handler + assert importlib.import_module(f"{OLD_PACKAGE}.adapters.transformation") is importlib.import_module( + f"{NEW_PACKAGE}.adapters.transformation" + ) + assert importlib.import_module(f"{OLD_PACKAGE}.safeguards") is importlib.import_module(f"{NEW_PACKAGE}.safeguards") + + +def test_old_package_attribute_access_reaches_renamed_subpackage(monkeypatch: pytest.MonkeyPatch) -> None: + _forget_old_package(monkeypatch) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + old_package: Final = importlib.import_module(OLD_PACKAGE) + + assert old_package.messages is importlib.import_module(f"{NEW_PACKAGE}.native") # pyright: ignore[reportAny] # module attributes are untyped + assert old_package.messages.handler is importlib.import_module(f"{NEW_PACKAGE}.native.handler") # pyright: ignore[reportAny] # module attributes are untyped + with pytest.raises(AttributeError, match="no attribute 'nope'"): + _ = old_package.nope # pyright: ignore[reportAny] # module attributes are untyped + + +def test_old_transformation_import_loads_only_its_new_counterpart(monkeypatch: pytest.MonkeyPatch) -> None: + _forget_old_package(monkeypatch) + monkeypatch.delitem(sys.modules, f"{NEW_PACKAGE}.native.streaming_iterator", raising=False) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + old_transformation: Final = importlib.import_module(f"{OLD_PACKAGE}.messages.transformation") + + assert old_transformation is sys.modules[f"{NEW_PACKAGE}.native.transformation"] + assert f"{NEW_PACKAGE}.native.streaming_iterator" not in sys.modules, "shim imported the streaming iterator" + + +def test_old_subpackage_import_keeps_the_new_package_path_importable(monkeypatch: pytest.MonkeyPatch) -> None: + _forget_old_package(monkeypatch) + monkeypatch.delitem(sys.modules, f"{NEW_PACKAGE}.native.mcp_handler", raising=False) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + native: Final = importlib.import_module(f"{OLD_PACKAGE}.messages") + + assert [path.rsplit("/", 2)[-2:] for path in native.__path__] == [["messages", "native"]] # pyright: ignore[reportAny] # module attributes are untyped + assert native.__spec__ is not None and native.__spec__.name == f"{NEW_PACKAGE}.native" + assert importlib.import_module(f"{NEW_PACKAGE}.native.mcp_handler").__name__ == f"{NEW_PACKAGE}.native.mcp_handler" diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 093b4d6cce8..eb5c1598c17 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -436,7 +436,7 @@ class TestPassthroughOAuth: def test_passthrough_oauth_no_x_api_key(self): """Passthrough endpoint should not add x-api-key for OAuth tokens.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -459,7 +459,7 @@ class TestPassthroughOAuth: def test_passthrough_regular_key_uses_x_api_key(self): """Passthrough endpoint should still use x-api-key for regular API keys.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1265,7 +1265,7 @@ class TestPassthroughAuthToken: """Passthrough endpoint should use Bearer auth when only ANTHROPIC_AUTH_TOKEN is set.""" from unittest.mock import patch as mock_patch - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1289,7 +1289,7 @@ class TestPassthroughAuthToken: """Passthrough endpoint should prefer ANTHROPIC_API_KEY over ANTHROPIC_AUTH_TOKEN.""" from unittest.mock import patch as mock_patch - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1320,7 +1320,7 @@ class TestPassthroughAuthToken: from unittest.mock import patch as mock_patch import litellm - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1342,7 +1342,7 @@ class TestPassthroughAuthToken: """A client-forwarded x-api-key header, whatever its casing, should satisfy validation without env credentials.""" from unittest.mock import patch as mock_patch - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1365,7 +1365,7 @@ class TestPassthroughAuthToken: """get_complete_url should use ANTHROPIC_BASE_URL when api_base is None.""" from unittest.mock import patch as mock_patch - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -2043,7 +2043,7 @@ class TestAnthropicThinkingSignatureSelfHeal: def test_anthropic_messages_config_http_retry_helpers(self): import httpx - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -2113,7 +2113,7 @@ class TestAnthropicThinkingSignatureSelfHeal: def test_anthropic_messages_retry_honors_thinking_block_binding( self, thinking: Mapping[str, object], error_message: str, expected: bool ) -> None: - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -2865,7 +2865,7 @@ class TestWifServerOwnedAuthHeaderStrip: """The passthrough surface hands the caller's own credential upstream rather than minting. That check was case-sensitive, so X-Api-Key slipped past it and the caller's key would have travelled beside a minted Bearer.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -3084,7 +3084,7 @@ class TestMessagesEnvAuthBetaMerge: def test_client_beta_survives_env_auth_injection(self, monkeypatch, clean_anthropic_env): """Regression: headers.update(auth_header) silently clobbered the client's anthropic-beta on the native /v1/messages route.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -3166,7 +3166,7 @@ class TestWifLitellmParamsPlumbing: assert len(poster.requests) == 1 def test_messages_mints_from_litellm_params(self, wif_engine): - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -3245,7 +3245,7 @@ class TestWifAsyncSeam: @pytest.mark.asyncio async def test_avalidate_messages_environment_mints_off_loop(self, wif_async_engine, monkeypatch): - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -3268,7 +3268,7 @@ class TestWifAsyncSeam: async def test_avalidate_delegates_to_subclass_sync_override(self): """A provider subclass that only overrides the sync method must keep its behavior when the handler goes through the async variant.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index df89774951f..7a6b6534131 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -13,7 +13,7 @@ import litellm from litellm.caching.dual_cache import DualCache from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.anthropic.count_tokens import handler as count_handler -from litellm.llms.anthropic.pass_through.messages.transformation import DEFAULT_ANTHROPIC_API_VERSION +from litellm.llms.anthropic.messages.native.transformation import DEFAULT_ANTHROPIC_API_VERSION from litellm.llms.anthropic.prompt_cache_prediction import ( CountedPromptCachePlan, NativePredictionTarget, diff --git a/tests/unit/llms/azure/decisions/__init__.py b/tests/unit/llms/azure/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/azure/decisions/test_transformation.py b/tests/unit/llms/azure/decisions/test_transformation.py new file mode 100644 index 00000000000..b6de6084b9c --- /dev/null +++ b/tests/unit/llms/azure/decisions/test_transformation.py @@ -0,0 +1,226 @@ +import json +import threading +import time +from collections.abc import Iterator, Mapping +from typing import Final +from unittest.mock import Mock + +import pytest +import respx +from azure.core.credentials import AccessToken + +import litellm +from litellm.llms.azure import common_utils +from litellm.llms.azure.decisions.transformation import AzureDecisionsConfig +from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse + +_BASE: Final = "https://resource.openai.azure.com" +_URL: Final = f"{_BASE}/openai/v1/decisions" +_RESPONSE: Final = { + "model": "luna-deployment", + "answers": [{"type": "predicate", "name": "defect", "probability": 0.9}], + "usage": {"input_tokens": 10, "output_tokens": 1, "total_tokens": 11}, +} + + +@pytest.fixture(autouse=True) +def clear_credentials(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + for name in ( + "AZURE_API_KEY", + "AZURE_OPENAI_API_KEY", + "AZURE_API_BASE", + "AZURE_AD_TOKEN", + "AZURE_CLIENT_ID", + "AZURE_TENANT_ID", + "AZURE_CLIENT_SECRET", + "AZURE_SCOPE", + "AZURE_USERNAME", + "AZURE_PASSWORD", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "azure_key", None) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "enable_azure_ad_token_refresh", False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + yield + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + + +async def call_decisions( + is_async: bool, + systemone: bool, + credentials: Mapping[str, object], + api_base: str = _BASE, +) -> DecisionsResponse | OpenAIDecisionResponse: + params = { + "model": "azure/luna-deployment", + "api_base": api_base, + **credentials, + } + if systemone: + params["state"] = "The export job hangs." + params["questions"] = {"defect": {"type": "noul", "instructions": "Is this a defect?"}} + if is_async: + return await litellm.asystemone(**params) + return litellm.systemone(**params) + params["input"] = "The export job hangs." + params["questions"] = [{"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}] + if is_async: + return await litellm.adecisions(**params) + return litellm.decisions(**params) + + +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.parametrize("systemone", (False, True)) +@pytest.mark.parametrize("suffix", ("", "/", "/openai", "/openai/v1/")) +async def test_azure_decisions_uses_entra_token_and_openai_wire_format( + respx_mock: respx.MockRouter, is_async: bool, systemone: bool, suffix: str +) -> None: + route = respx_mock.post(_URL).respond(json=_RESPONSE) + response = await call_decisions(is_async, systemone, {"azure_ad_token": "synthetic-entra"}, _BASE + suffix) + request = route.calls.last.request + assert request.headers["authorization"] == "Bearer synthetic-entra" + assert "api-key" not in request.headers + assert json.loads(request.content) == { + "model": "luna-deployment", + "input": "The export job hangs.", + "questions": [{"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}], + } + if systemone: + assert response.model_dump()["answers"]["defect"]["noul"] == 0.9 + else: + assert response.model_dump()["answers"][0]["probability"] == 0.9 + assert response._hidden_params["custom_llm_provider"] == "azure" + + +@pytest.mark.parametrize("is_async", (False, True)) +@pytest.mark.parametrize("systemone", (False, True)) +async def test_default_credential_refreshes_without_opt_in( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + is_async: bool, + systemone: bool, +) -> None: + credential = Mock(spec=["get_token"]) + credential.get_token.side_effect = [ + AccessToken("synthetic-first", int(time.time()) + 120), + AccessToken("synthetic-refreshed", int(time.time()) + 3600), + ] + factory = Mock(return_value=credential) + monkeypatch.setattr("azure.identity.DefaultAzureCredential", factory) + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-unrelated-openai-key") + route = respx_mock.post(_URL).respond(json=_RESPONSE) + await call_decisions(is_async, systemone, {}) + await call_decisions(is_async, systemone, {}) + assert [call.request.headers["authorization"] for call in route.calls] == [ + "Bearer synthetic-first", + "Bearer synthetic-refreshed", + ] + factory.assert_called_once_with() + assert credential.get_token.call_count == 2 + assert all( + call.args == ("https://cognitiveservices.azure.com/.default",) for call in credential.get_token.call_args_list + ) + + +@pytest.mark.parametrize("kind", ("service-principal", "oidc", "provider")) +async def test_azure_decisions_reuses_existing_azure_credential_resolution( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, kind: str +) -> None: + provider = Mock(return_value="synthetic-entra") + if kind == "service-principal": + factory = Mock(return_value=provider) + monkeypatch.setattr(common_utils, "get_azure_ad_token_from_entra_id", factory) + credentials = { + "tenant_id": "tenant", + "client_id": "client", + "client_secret": "synthetic-secret", + } + elif kind == "oidc": + monkeypatch.setattr(common_utils, "get_azure_ad_token_from_oidc", provider) + credentials = { + "tenant_id": "tenant", + "client_id": "client", + "azure_ad_token": "oidc/azure/https://cognitiveservices.azure.com/.default", + } + else: + credentials = {"azure_ad_token_provider": provider} + route = respx_mock.post(_URL).respond(json=_RESPONSE) + await call_decisions(True, False, credentials) + assert route.calls.last.request.headers["authorization"] == "Bearer synthetic-entra" + provider.assert_called_once() + if kind == "service-principal": + factory.assert_called_once_with( + tenant_id="tenant", + client_id="client", + client_secret="synthetic-secret", + scope="https://cognitiveservices.azure.com/.default", + ) + if kind == "oidc": + provider.assert_called_once_with( + azure_ad_token=credentials["azure_ad_token"], + azure_client_id="client", + azure_tenant_id="tenant", + scope="https://cognitiveservices.azure.com/.default", + ) + + +@pytest.mark.parametrize("key_source", ("api_key", "AZURE_API_KEY", "AZURE_OPENAI_API_KEY")) +async def test_azure_api_key_takes_precedence_over_entra( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, key_source: str +) -> None: + provider = Mock(side_effect=AssertionError("API key must bypass Entra")) + credentials = {"azure_ad_token_provider": provider} + if key_source == "api_key": + credentials["api_key"] = "synthetic-key" + else: + monkeypatch.setenv(key_source, "synthetic-key") + route = respx_mock.post(_URL).respond(json=_RESPONSE) + await call_decisions(True, False, credentials) + assert route.calls.last.request.headers["api-key"] == "synthetic-key" + assert "authorization" not in route.calls.last.request.headers + provider.assert_not_called() + + +async def test_azure_credential_lookup_does_not_block_the_event_loop( + respx_mock: respx.MockRouter, +) -> None: + event_loop_thread = threading.get_ident() + + def provider() -> str: + assert threading.get_ident() != event_loop_thread + return "synthetic-entra" + + respx_mock.post(_URL).respond(json=_RESPONSE) + await call_decisions(True, False, {"azure_ad_token_provider": provider}) + + +def test_azure_decisions_requires_azure_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_API_BASE", "https://unrelated.example") + with pytest.raises(litellm.BadRequestError, match="api_base is required"): + litellm.decisions( + model="azure/luna-deployment", + input="Text", + questions=[ + { + "type": "predicate", + "name": "defect", + "instructions": "Is this a defect?", + } + ], + ) + + +def test_azure_decisions_does_not_echo_invalid_credentials() -> None: + with pytest.raises(litellm.BadRequestError) as error: + AzureDecisionsConfig().validate_environment( + {}, + "luna-deployment", + None, + {"azure_ad_token": {"secret": "sensitive-value"}}, + ) + assert "sensitive-value" not in str(error.value) diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index 3aebf56ee73..798cbe5fe00 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -355,3 +355,49 @@ def test_invalid_provider_response_is_an_api_error(payload: object, provider: st with pytest.raises(BaseLLMException, match="unexpected response") as error: config.transform_decisions_response("drex-v2.0", provider, httpx.Response(200, json=payload), request) assert error.value.status_code == 500 + + +@pytest.mark.parametrize( + ("probabilities", "expected"), + ( + ({}, ()), + ({"1": 0.8}, ((1, "medium", 0.8),)), + ({"0": 0.0, "1": 0.8}, ((0, "low", 0.0), (1, "medium", 0.8))), + ({"2": 0.0}, ((2, "high", 0.0),)), + ), + ids=("empty", "missing", "explicit_zero", "only_zero"), +) +def test_score_probabilities_preserve_missing_entries_and_explicit_zeros( + probabilities: Mapping[str, float], expected: tuple[tuple[int, str, float], ...] +) -> None: + ir: Final = systemone_request_to_ir( + _SYSTEMONE_BODY.validate_python( + { + "state": "Classify the fixture.", + "questions": {"score": {"type": "score", "criteria": ["low", "medium", "high"]}}, + } + ) + ) + payload: Final = { + "model": "fixture", + "answers": { + "score": { + "type": "score", + "score": 1, + "confidence": 0.8, + "legend": {"0": "low", "1": "medium", "2": "high"}, + "probabilities": probabilities, + } + }, + "usage": {"input_tokens": 0, "output_tokens": 0}, + } + + response: Final = parse_systemone_response(payload, ir) + answer: Final = response.answers[0] + + assert isinstance(answer, DecisionsIRScoreAnswer) + assert tuple((item.value, item.label, item.probability) for item in answer.probabilities) == expected + assert ( + ir_to_systemone_response(response, ir).model_dump(mode="json")["answers"]["score"]["probabilities"] + == probabilities + ) diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py index 4d9feea4454..93367fdff16 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude_transformation.py @@ -29,7 +29,7 @@ from litellm.constants import ( # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import ( +from litellm.llms.anthropic.messages.native.mid_conversation_system import ( as_system_content_blocks, ) from litellm.llms.bedrock.common_utils import ( diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 55fad96ceb1..297e64629a8 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -979,7 +979,7 @@ async def test_async_anthropic_messages_handler_streaming_forwards_provider_resp """ from collections.abc import AsyncIterator as ABCAsyncIterator - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1042,10 +1042,10 @@ async def test_async_anthropic_messages_handler_agentic_streaming_forwards_provi from collections.abc import AsyncIterator as ABCAsyncIterator from litellm.integrations.custom_logger import CustomLogger - from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) @@ -1113,7 +1113,7 @@ async def test_anthropic_messages_streaming_response_aclose_closes_upstream_stre the upstream stream so provider connections are released on client disconnect instead of lingering until garbage collection. """ - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, ) @@ -1145,10 +1145,10 @@ async def test_anthropic_messages_streaming_response_aclose_closes_upstream_stre @pytest.mark.asyncio async def test_anthropic_messages_streaming_response_aclose_closes_agentic_upstream_stream(): - from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( + from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) - from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( + from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, ) @@ -2096,7 +2096,7 @@ async def test_anthropic_post_skips_the_thinking_strip_retry_when_block_binding_ ) -> None: """A caller that pinned ``prefix_mismatch_behavior: error`` asked for the 400, so the handler must surface it instead of stripping the bound blocks and retrying.""" - from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig + from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig handler = BaseLLMHTTPHandler() thinking_block = {"type": "thinking", "thinking": "", "signature": "sig"} diff --git a/tests/unit/llms/databricks/decisions/test_transformation.py b/tests/unit/llms/databricks/decisions/test_transformation.py index f6ea3444db1..121c952cda9 100644 --- a/tests/unit/llms/databricks/decisions/test_transformation.py +++ b/tests/unit/llms/databricks/decisions/test_transformation.py @@ -1,6 +1,7 @@ from typing import Final import pytest +import responses from litellm.llms.databricks.decisions.transformation import DatabricksDecisionsConfig from litellm.types.decisions import ( @@ -21,6 +22,40 @@ _REQUEST: Final = DecisionsIRRequest( _QUESTIONS: Final = {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}} +@responses.activate +def test_should_resolve_oauth_credentials_without_a_static_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DATABRICKS_CLIENT_ID", "synthetic-client") + monkeypatch.setenv("DATABRICKS_CLIENT_SECRET", "synthetic-secret") + token: Final = responses.post(f"{_HOST}/oidc/v1/token", json={"access_token": "oauth-token", "expires_in": 3600}) + + assert DatabricksDecisionsConfig().resolve_credential(None, _BASE, {}) == "oauth-token" + assert token.call_count == 1 + + +@responses.activate +@pytest.mark.parametrize("key_source", ("argument", "DATABRICKS_API_KEY", "DATABRICKS_TOKEN")) +def test_should_preserve_static_key_precedence(monkeypatch: pytest.MonkeyPatch, key_source: str) -> None: + monkeypatch.setenv("DATABRICKS_CLIENT_ID", "synthetic-client") + monkeypatch.setenv("DATABRICKS_CLIENT_SECRET", "synthetic-secret") + if key_source != "argument": + monkeypatch.setenv(key_source, "static-key") + config: Final = DatabricksDecisionsConfig() + base, key = config.resolve_connection(_BASE, "static-key" if key_source == "argument" else None) + + assert base == _BASE + assert config.resolve_credential(key, _BASE, {}) == "static-key" + assert len(responses.calls) == 0 + + +@pytest.mark.parametrize("missing", ("DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET")) +def test_should_leave_incomplete_oauth_credentials_unresolved(monkeypatch: pytest.MonkeyPatch, missing: str) -> None: + monkeypatch.setenv("DATABRICKS_CLIENT_ID", "synthetic-client") + monkeypatch.setenv("DATABRICKS_CLIENT_SECRET", "synthetic-secret") + monkeypatch.delenv(missing) + + assert DatabricksDecisionsConfig().resolve_credential(None, _BASE, {}) is None + + @pytest.mark.parametrize("api_base", [_BASE, f"{_BASE}/"]) def test_a_serving_endpoint_name_is_called_on_its_invocations_route(api_base: str) -> None: assert _CONFIG.get_complete_url(api_base, "openjev") == f"{_BASE}/openjev/invocations" diff --git a/tests/unit/llms/databricks/test_databricks_common_utils.py b/tests/unit/llms/databricks/test_databricks_common_utils.py index 8f2d87a6d25..58aae2c1056 100644 --- a/tests/unit/llms/databricks/test_databricks_common_utils.py +++ b/tests/unit/llms/databricks/test_databricks_common_utils.py @@ -1,19 +1,21 @@ import json import sys +from functools import partial from types import ModuleType, SimpleNamespace from typing import Final +from unittest.mock import MagicMock, Mock, patch import httpx import pytest +import responses import respx from fastapi.testclient import TestClient -from unittest.mock import MagicMock, patch - import litellm from litellm._version import version +from litellm.caching.in_memory_cache import InMemoryCache from litellm.exceptions import BadRequestError -from litellm.llms.databricks.common_utils import DatabricksBase +from litellm.llms.databricks.common_utils import DatabricksBase, DatabricksException def test_databricks_validate_environment(): @@ -195,3 +197,79 @@ def test_embeddings_uses_databricks_sdk_if_api_key_and_base_not_specified( {"model": "bge-large-en-v1.5", "input": inputs, "extraparam": "testpassingextraparam"}, ) assert response.to_dict() == upstream_response + + +@responses.activate +def test_should_reuse_the_token_until_its_refresh_margin() -> None: + clock: Final = Mock(return_value=1000.0) + get_token: Final = partial( + DatabricksBase().get_cached_oauth_m2m_token, + client_id="synthetic-client", + client_secret="synthetic-secret", + token_cache=InMemoryCache(clock=clock), + ) + responses.post(f"{SDK_HOST}/oidc/v1/token", json={"access_token": "first-token", "expires_in": 3600}) + responses.post(f"{SDK_HOST}/oidc/v1/token", json={"access_token": "refreshed-token", "expires_in": 3600}) + + assert get_token(SDK_HOST) == "first-token" + clock.return_value = 4539.0 + assert get_token(f"{SDK_HOST}/serving-endpoints/") == "first-token" + assert len(responses.calls) == 1 + clock.return_value = 4541.0 + assert get_token(SDK_HOST) == "refreshed-token" + assert len(responses.calls) == 2 + + +@responses.activate +@pytest.mark.parametrize("expires_in", (0, 30, 60)) +def test_should_not_cache_a_token_within_the_refresh_margin(expires_in: int) -> None: + get_token: Final = partial( + DatabricksBase().get_cached_oauth_m2m_token, + SDK_HOST, + "synthetic-client", + "synthetic-secret", + token_cache=InMemoryCache(), + ) + token: Final = responses.post( + f"{SDK_HOST}/oidc/v1/token", json={"access_token": "short-lived-token", "expires_in": expires_in} + ) + + assert get_token() == "short-lived-token" + assert get_token() == "short-lived-token" + assert token.call_count == 2 + + +@responses.activate +@pytest.mark.parametrize("changed", ("workspace", "client_id", "client_secret")) +def test_should_not_reuse_a_token_for_a_different_workspace_or_credentials(changed: str) -> None: + base: Final = DatabricksBase() + cache: Final = InMemoryCache() + responses.post(f"{SDK_HOST}/oidc/v1/token", json={"access_token": "original-token", "expires_in": 3600}) + assert ( + base.get_cached_oauth_m2m_token(SDK_HOST, "synthetic-client", "synthetic-secret", token_cache=cache) + == "original-token" + ) + + next_host: Final = "https://other-workspace.example" if changed == "workspace" else SDK_HOST + client_id: Final = "rotated-client" if changed == "client_id" else "synthetic-client" + client_secret: Final = "rotated-secret" if changed == "client_secret" else "synthetic-secret" + responses.post(f"{next_host}/oidc/v1/token", json={"access_token": "new-token", "expires_in": 3600}) + assert base.get_cached_oauth_m2m_token(next_host, client_id, client_secret, token_cache=cache) == "new-token" + assert len(responses.calls) == 2 + + +@responses.activate +def test_should_propagate_oauth_failure_and_retry_the_next_request() -> None: + get_token: Final = partial( + DatabricksBase().get_cached_oauth_m2m_token, + SDK_HOST, + "synthetic-client", + "synthetic-secret", + token_cache=InMemoryCache(), + ) + responses.post(f"{SDK_HOST}/oidc/v1/token", status=401, json={"error": "invalid_client"}) + responses.post(f"{SDK_HOST}/oidc/v1/token", json={"access_token": "valid-token", "expires_in": 3600}) + + with pytest.raises(DatabricksException, match="OAuth M2M token request failed"): + get_token() + assert get_token() == "valid-token" diff --git a/tests/unit/llms/deepseek/messages/test_deepseek_anthropic_messages_transformation.py b/tests/unit/llms/deepseek/messages/test_deepseek_anthropic_messages_transformation.py index 0afe57a001f..316b8576ac4 100644 --- a/tests/unit/llms/deepseek/messages/test_deepseek_anthropic_messages_transformation.py +++ b/tests/unit/llms/deepseek/messages/test_deepseek_anthropic_messages_transformation.py @@ -1,5 +1,5 @@ import litellm -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.deepseek.messages.transformation import ( diff --git a/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py b/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py index 46062f16c9a..20ff65f11d9 100644 --- a/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py +++ b/tests/unit/llms/github_copilot/messages/test_github_copilot_messages_transformation.py @@ -334,7 +334,7 @@ def test_github_copilot_config_does_not_handle_web_search_natively(): interception handler short-circuiting Copilot instead of routing to it, even though Copilot now has a BaseAnthropicMessagesConfig. The base Anthropic config (bedrock/vertex/anthropic path) must report True.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/unit/llms/microsoft_365_copilot/chat/test_transformation.py b/tests/unit/llms/microsoft_365_copilot/chat/test_transformation.py index 22cb8da3aed..aab3c82a389 100644 --- a/tests/unit/llms/microsoft_365_copilot/chat/test_transformation.py +++ b/tests/unit/llms/microsoft_365_copilot/chat/test_transformation.py @@ -5,7 +5,7 @@ import pytest from pydantic import TypeAdapter import litellm -from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter +from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter from litellm.llms.microsoft_365_copilot.chat.transformation import ( Microsoft365CopilotChatConfig, build_chat_request, diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index 2a07462fe81..eddfe578256 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -299,3 +299,55 @@ def test_an_openai_response_becomes_systemone_answers_with_the_callers_score_lab } assert response.usage is not None assert (response.usage.cached_tokens, response.usage.cache_write_tokens) == (256, 64) + + +@pytest.mark.parametrize( + ("choice_probabilities", "score_probabilities"), + ( + ((), ()), + ( + ({"value": True, "probability": 0.8},), + ({"value": 1, "label": "major", "probability": 0.8},), + ), + ( + ( + {"value": True, "probability": 0.0}, + {"value": "escalate", "probability": 0.8}, + ), + ( + {"value": 0, "label": "minor", "probability": 0.0}, + {"value": 1, "label": "major", "probability": 0.8}, + ), + ), + ( + ({"value": True, "probability": 0.0},), + ({"value": 1, "label": "major", "probability": 0.0},), + ), + ), + ids=("empty", "missing", "explicit_zero", "only_zero"), +) +def test_openai_probabilities_preserve_missing_entries_and_explicit_zeros( + choice_probabilities: tuple[Mapping[str, object], ...], + score_probabilities: tuple[Mapping[str, object], ...], +) -> None: + ir: Final = _openai_ir(_OPENAI_REQUEST) + payload: Final = { + **_OPENAI_RESPONSE, + "answers": [ + _PREDICATE_ANSWER, + {**_CHOICE_ANSWER, "probabilities": choice_probabilities}, + { + "type": "score", + "name": "severity", + "score": 1, + "confidence": 0.8, + "probabilities": score_probabilities, + }, + _REFUSAL_ANSWER, + ], + } + + response: Final = ir_to_openai_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir, "requested") + + assert response.answers[1].model_dump(mode="json")["probabilities"] == list(choice_probabilities) + assert response.answers[2].model_dump(mode="json")["probabilities"] == list(score_probabilities) diff --git a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py index 8995b3cb068..d5e7bb4f93b 100644 --- a/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_count_tokens_transformation.py @@ -2,7 +2,7 @@ from typing import Final import pytest -from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( +from litellm.llms.anthropic.messages.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.llms.openai.responses.count_tokens.transformation import ( diff --git a/tests/unit/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py b/tests/unit/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py index 1d5ef50ca3a..935e3e4a21d 100644 --- a/tests/unit/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py +++ b/tests/unit/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py @@ -396,7 +396,7 @@ def test_request_defaults_missing_cache_control_type_and_drops_non_dict(config): def test_native_anthropic_config_keeps_cache_control_ttl(): """Anthropic itself accepts ttl, so the normalization must stay scoped to the OpenAI-like passthrough and never reach the native Anthropic path.""" - from litellm.llms.anthropic.pass_through.messages.transformation import ( + from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) diff --git a/tests/unit/llms/tencent/messages/test_tencent_anthropic_messages_transformation.py b/tests/unit/llms/tencent/messages/test_tencent_anthropic_messages_transformation.py index e5bdce9d3e0..b737ad3e9c1 100644 --- a/tests/unit/llms/tencent/messages/test_tencent_anthropic_messages_transformation.py +++ b/tests/unit/llms/tencent/messages/test_tencent_anthropic_messages_transformation.py @@ -1,5 +1,5 @@ import litellm -from litellm.llms.anthropic.pass_through.messages.transformation import ( +from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.tencent.messages.transformation import ( diff --git a/tests/unit/llms/test_get_complete_url_trailing_slash.py b/tests/unit/llms/test_get_complete_url_trailing_slash.py index 9118498c2c2..cd778c20d9b 100644 --- a/tests/unit/llms/test_get_complete_url_trailing_slash.py +++ b/tests/unit/llms/test_get_complete_url_trailing_slash.py @@ -145,7 +145,7 @@ API_BASE_IS_THE_FULL_URL: Final[Mapping[str, str]] = MappingProxyType( "searchapi.search.transformation.SearchAPIConfig", "tinyfish.search.transformation.TinyfishSearchConfig", "vertex_ai.text_to_speech.transformation.VertexAITextToSpeechConfig", - "vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation." + "vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation." "VertexAIPartnerModelsAnthropicMessagesConfig", ) } diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index a1643f0f699..b2a0b735b0f 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -14,7 +14,7 @@ import pytest from pydantic import JsonValue, TypeAdapter import litellm -from litellm.llms.anthropic.pass_through.adapters.transformation import ( +from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, truncate_tool_name, ) @@ -622,7 +622,7 @@ async def test_anthropic_view_applies_clear_edits_without_calling_the_compaction with ( patch( - "litellm.llms.anthropic.pass_through.context_management.editors.compact._read_summary_model_setting", + "litellm.llms.anthropic.messages.context_management.editors.compact._read_summary_model_setting", return_value="gemini/gemini-2.5-flash", ), patch.object(litellm, "acompletion", side_effect=AssertionError("count_tokens must not summarize")) as summary, diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py index 75449eacfd3..f3741d2667a 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_ai_gemini_transformation.py @@ -2842,7 +2842,7 @@ def test_thinking_block_signature_is_not_forwarded_to_gemini() -> None: def test_anthropic_messages_history_replays_to_gemini_without_claude_signature() -> None: from litellm.litellm_core_utils.prompt_templates.factory import _get_dummy_thought_signature - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter thinking: Final = "I should look the weather up before answering." anthropic_messages: Final = [ diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 3bc864cc047..0f5c21d08c7 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -20,7 +20,7 @@ from litellm.cost_calculator import batch_cost_calculator from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details -from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler +from litellm.llms.anthropic.messages.native import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.llms.vertex_ai.common_utils import GEMINI_MAX_ENUM_CHARACTERS, VertexAIError @@ -7544,7 +7544,7 @@ def test_gemini_candidate_with_finish_reason_no_content_chat_completion(): def test_gemini_candidate_with_finish_reason_no_content_anthropic_messages(): - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) @@ -7617,7 +7617,7 @@ def test_gemini_candidate_with_finish_reason_no_content_responses_api(): def test_gemini_candidate_other_finish_reasons_no_content(): - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, ) from litellm.responses.litellm_completion_transformation.transformation import ( diff --git a/tests/unit/llms/vertex_ai/test_vertex.py b/tests/unit/llms/vertex_ai/test_vertex.py index 04cf391c700..7e0baaf9e10 100644 --- a/tests/unit/llms/vertex_ai/test_vertex.py +++ b/tests/unit/llms/vertex_ai/test_vertex.py @@ -2461,7 +2461,7 @@ async def test_anthropic_message_via_anthropic_messages( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) from litellm.utils import ProviderConfigManager diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/test_experimental_pass_through_shim.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/test_experimental_pass_through_shim.py new file mode 100644 index 00000000000..4860b5dbe7e --- /dev/null +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/test_experimental_pass_through_shim.py @@ -0,0 +1,20 @@ +import importlib +import sys +from typing import Final + +import pytest + +OLD_PACKAGE: Final = "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through" +NEW_PACKAGE: Final = "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages" + + +def test_old_deep_import_is_the_new_module_object_and_warns(monkeypatch: pytest.MonkeyPatch) -> None: + for name in tuple(sys.modules): + if name == OLD_PACKAGE or name.startswith(OLD_PACKAGE + "."): + monkeypatch.delitem(sys.modules, name) + with pytest.warns(DeprecationWarning, match=NEW_PACKAGE): + old_transformation = importlib.import_module(f"{OLD_PACKAGE}.transformation") + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages import transformation + + assert old_transformation is transformation + assert sys.modules[f"{OLD_PACKAGE}.transformation"] is sys.modules[f"{NEW_PACKAGE}.transformation"] diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index ae9b9684f2d..6c10d524531 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -2,18 +2,98 @@ import copy import json import os from collections.abc import Mapping +from datetime import datetime, timedelta +from queue import SimpleQueue +from threading import get_ident from typing import Final from unittest.mock import MagicMock, patch import pytest +import requests +import responses -from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( +from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) from litellm.llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS from litellm.types.router import GenericLiteLLMParams +@pytest.mark.parametrize("stream", (False, True)) +@pytest.mark.parametrize("infer_project", (False, True), ids=("explicit-project", "credential-project")) +@pytest.mark.asyncio +async def test_messages_auth_load_and_refresh_leave_the_event_loop(stream: bool, infer_project: bool) -> None: + config: Final = VertexAIPartnerModelsAnthropicMessagesConfig() + loop_thread: Final = get_ident() + token_threads: Final[SimpleQueue[int]] = SimpleQueue() + now: Final = datetime(2025, 1, 1) + model: Final = "claude-sonnet-5" + headers: Final = {"Authorization": "Bearer incoming", "anthropic-beta": "retained-beta", "x-request-id": "req-1"} + optional_params: Final = { + "stream": stream, + "tools": [{"type": ANTHROPIC_HOSTED_TOOLS.WEB_SEARCH.value, "name": "web_search"}], + } + litellm_params: Final = { + "vertex_ai_project": None if infer_project else "auth-project", + "vertex_ai_location": "global", + "vertex_credentials": json.dumps( + { + "type": "authorized_user", + "client_id": "synthetic-client", + "client_secret": "synthetic-secret", + "refresh_token": "synthetic-refresh", + "quota_project_id": "auth-project", + } + ), + } + + async def validate() -> tuple[Mapping[str, object], str | None]: + return await config.avalidate_anthropic_messages_environment( + headers=headers, model=model, messages=[], optional_params=optional_params, litellm_params=litellm_params + ) + + with patch("google.auth._helpers.utcnow", return_value=now) as clock, responses.RequestsMock() as oauth: + + def token_response(_request: requests.PreparedRequest) -> tuple[int, dict[str, str], str]: + token_threads.put(get_ident()) + return 200, {}, json.dumps({"access_token": f"token-{len(oauth.calls) + 1}", "expires_in": 3600}) + + oauth.add_callback(responses.POST, "https://oauth2.googleapis.com/token", callback=token_response) + cold_headers, cold_url = await validate() + assert token_threads.get_nowait() != loop_thread + assert cold_headers["Authorization"] == "Bearer token-1" + assert await validate() == (cold_headers, cold_url) + assert len(oauth.calls) == 1 + + clock.return_value = now + timedelta(hours=2) + refreshed_headers, refreshed_url = await validate() + assert token_threads.get_nowait() != loop_thread + assert refreshed_headers["Authorization"] == "Bearer token-2" + assert refreshed_url == cold_url + assert len(oauth.calls) == 2 + assert config.validate_anthropic_messages_environment( + headers=headers, model=model, messages=[], optional_params=optional_params, litellm_params=litellm_params + ) == (refreshed_headers, refreshed_url) + assert len(oauth.calls) == 2 + + endpoint: Final = "streamRawPredict" if stream else "rawPredict" + assert cold_url == ( + "https://aiplatform.googleapis.com/v1/projects/auth-project/locations/global/" + f"publishers/anthropic/models/{model}:{endpoint}" + ) + assert refreshed_headers == { + "Authorization": "Bearer token-2", + "anthropic-beta": cold_headers["anthropic-beta"], + "x-request-id": "req-1", + "content-type": "application/json", + } + beta_header: Final = cold_headers["anthropic-beta"] + assert isinstance(beta_header, str) + assert set(beta_header.split(",")) == {"retained-beta", ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value} + assert headers == {"Authorization": "Bearer incoming", "anthropic-beta": "retained-beta", "x-request-id": "req-1"} + + def test_validate_environment_uses_vertex_ai_location(): config = VertexAIPartnerModelsAnthropicMessagesConfig() headers = {} diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 4384507e841..a0a79f9b7cf 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -1784,3 +1784,53 @@ def test_a_whole_number_request_size_limit_is_named_as_configured_in_the_413( assert response.status_code == 413, response.text assert response.json() == {"error": "Request size is too large. Max size is 1 MB"} assert not upstream.called + + +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions")) +@pytest.mark.parametrize("api_key", (None, "deployment-key")) +def test_proxy_routes_lm_studio_decisions_with_optional_authentication( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, + api_key: str | None, +) -> None: + monkeypatch.delenv("LM_STUDIO_API_KEY", raising=False) + deployment_router: Final = litellm.Router( + model_list=[ + { + "model_name": "local-decision", + "litellm_params": { + "model": "lm_studio/served-decision", + "api_base": "http://lmstudio.local:1234/v1", + "api_key": api_key, + }, + } + ] + ) + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", deployment_router) + upstream: Final = respx_mock.post("http://lmstudio.local:1234/v1/systemone").respond(json=_RESPONSE) + fields: Final = ( + {"state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}} + if endpoint.endswith("systemone") + else { + "input": "review", + "questions": [{"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}], + } + ) + + response: Final = client.post(endpoint, json={"model": "local-decision", **fields}) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls.last.request.content) == { + "model": "served-decision", + "state": "review", + "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + } + assert upstream.calls.last.request.headers.get("authorization") == (f"Bearer {api_key}" if api_key else None) + expected: Final = ( + _RESPONSE["answers"] + if endpoint.endswith("systemone") + else [{"type": "predicate", "name": "is_defect", "probability": 0.9}] + ) + assert response.json()["answers"] == expected diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 69b3e113b8f..8c8aa7af19f 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -3258,7 +3258,7 @@ async def test_streaming_hook_fails_closed_when_assembler_raises_api_error(): guardrail = _sse_guardrail() with patch( - "litellm.llms.anthropic.pass_through.stream_assembly.build_complete_streaming_response", + "litellm.llms.anthropic.messages.stream_assembly.build_complete_streaming_response", side_effect=litellm.APIError( status_code=500, message="Error building chunks for logging/streaming usage calculation", diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py index 5bdcdddf4b7..3336b20f5eb 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -2711,7 +2711,7 @@ async def test_retrieved_content_protected_when_mcp_tool_name_is_truncated(guard the OpenAI-translated view the guardrail scans, dropping the suffix. The call id read from the request's own Anthropic tool_use (never truncated) still pairs the retrieved row so it is held back.""" - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( truncate_tool_name, ) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 7530703a05a..31a963d3e47 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -11,7 +11,7 @@ from pydantic import ValidationError import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.stream_assembly import extract_sse_data +from litellm.llms.anthropic.messages.stream_assembly import extract_sse_data from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import ( AnthropicPassthroughLoggingHandler, ) diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index 4aa7e9cfbae..ae95c27950c 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -20,7 +20,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.tinyfish_passthr from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclose_superseded_attempt +from litellm.llms.anthropic.messages.native.streaming_iterator import aclose_superseded_attempt from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 78f5e1ace0c..86d064050eb 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -5074,7 +5074,7 @@ async def test_spend_log_request_id_is_the_message_id_a_bridged_streaming_caller adapter mints itself, and it is the only request id that call ever shows the caller, so GET /spend/logs?request_id=msg_... has to land on the row.""" from litellm.litellm_core_utils.litellm_logging import Logging - from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import ( + from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) from litellm.types.llms.openai import ( diff --git a/tests/unit/proxy/test_budget_reservation.py b/tests/unit/proxy/test_budget_reservation.py index 4635e145914..42b41a2e3f2 100644 --- a/tests/unit/proxy/test_budget_reservation.py +++ b/tests/unit/proxy/test_budget_reservation.py @@ -13,10 +13,10 @@ import litellm from litellm.caching.dual_cache import DualCache from litellm.types.caching import RedisPipelineIncrementOperation from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES -from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( +from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( AgenticAnthropicStreamingIterator, ) -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( +from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, ) from litellm.proxy._types import ( diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 82dd3479321..5c99e5a01ea 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -8640,10 +8640,10 @@ class TestStreamingClientDisconnectBilling: through the translate_completion_output_params_streaming result to the inner chat stream's collected chunks or a disconnect bills nothing. """ - from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( + from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicSSEStream, ) - from litellm.llms.anthropic.pass_through.adapters.transformation import ( + from litellm.llms.anthropic.messages.adapters.transformation import ( AnthropicAdapter, ) from litellm.router import FallbackAwareAnthropicMessagesStream @@ -9180,7 +9180,7 @@ def _apply_stream_usage_tracking( @pytest.mark.asyncio async def test_disconnect_bills_partial_default_route_messages_stream_through_the_responses_adapter(self): - from litellm.llms.anthropic.pass_through.responses_adapters.streaming_iterator import ( + from litellm.llms.anthropic.messages.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) from litellm.router import FallbackAwareAnthropicMessagesStream @@ -10175,6 +10175,32 @@ class TestInjectCostIntoUsageDict: class TestProcessChunkWithCostInjection: + @pytest.mark.parametrize("model", ["claude-opus-5-5", "vertex_ai/claude-opus-5-5"]) + @pytest.mark.parametrize("search_requests", [None, "null", 0, 1, 2]) + def test_claude_search_stream_cost_matches_complete_usage( + self, monkeypatch: pytest.MonkeyPatch, model: str, search_requests: int | Literal["null"] | None + ) -> None: + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) + usage: Final = litellm.Usage( + prompt_tokens=10760, + completion_tokens=399, + total_tokens=11159, + server_tool_use={"web_search_requests": search_requests} if isinstance(search_requests, int) else None, + ) + wire_usage: Final = usage.model_dump() | ({"server_tool_use": None} if search_requests == "null" else {}) + event: Final = {"object": "chat.completion.chunk", "choices": [], "usage": wire_usage} + chunk: Final = f"data: {json.dumps(event)}\n\n".encode() + + result: Final = ProxyBaseLLMRequestProcessing.process_chunk_with_cost_injection(chunk, model) + + assert isinstance(result, bytes) + payload: Final = json.loads(result.decode().removeprefix("data: ")) + expected_cost: Final = litellm.completion_cost( + model=model, completion_response=litellm.ModelResponse(usage=usage) + ) + assert payload["usage"]["cost"] == pytest.approx(expected_cost) + assert "cost" not in event["usage"] + def test_complete_usage_frame_chunk_is_injected(self, monkeypatch): monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) chunk = ( @@ -11897,7 +11923,7 @@ class TestResponseCostFromLoggingObjForBridgedMessages: def test_recomputed_cost_prices_the_reasoning_tokens_of_the_bridged_chat_completion(self): from litellm.types.utils import CompletionTokensDetailsWrapper, ModelResponse, Usage - from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter model = "dashscope/qwen-turbo" pricing = litellm.get_model_info(model) diff --git a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 57b68e6db05..4360655c8ca 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -1359,35 +1359,45 @@ class TestProxySettingEndpoints: assert values["logo_url"] == "https://cdn.example.com/logo.png" assert values["favicon_url"] == "https://cdn.example.com/favicon.ico" + @pytest.mark.parametrize("env_logo", [ + "https://env.example.com/logo.png", "/mnt/secret/internal/logo.png", + ]) def test_get_ui_theme_settings_stored_value_wins_over_env( - self, mock_auth, monkeypatch - ): + self, monkeypatch: pytest.MonkeyPatch, env_logo: str, + ) -> None: """A stored ui_theme_config field outranks the env var for that field. The env fallback only fills fields the stored config leaves blank, so the UI-driven flow is unchanged while an unstored field still resolves. """ from litellm.proxy.proxy_server import proxy_config + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import UIThemeSettingsResponse - stored_config = { + stored_config: Final[dict[str, object]] = { "litellm_settings": { - "ui_theme_config": {"logo_url": "https://db.example.com/logo.png"} + "ui_theme_config": { + "logo_url": "https://db.example.com/logo.png", + "logo_url_dark": "https://db.example.com/logo-dark.png", + } } } - async def mock_get_config(): + async def mock_get_config() -> dict[str, object]: return stored_config monkeypatch.setattr(proxy_config, "get_config", mock_get_config) - monkeypatch.setenv("UI_LOGO_PATH", "https://env.example.com/logo.png") + monkeypatch.setenv("UI_LOGO_PATH", env_logo) + monkeypatch.setenv("UI_LOGO_PATH_DARK", env_logo) monkeypatch.setenv("LITELLM_FAVICON_URL", "https://env.example.com/favicon.ico") - response = client.get("/get/ui_theme_settings") + response: Final = client.get("/get/ui_theme_settings") assert response.status_code == 200 - values = response.json()["values"] - assert values["logo_url"] == "https://db.example.com/logo.png" - assert values["favicon_url"] == "https://env.example.com/favicon.ico" + result: Final = UIThemeSettingsResponse.model_validate(response.json()) + assert result.values["logo_url"] == "https://db.example.com/logo.png" + assert result.values["logo_url_dark"] == "https://db.example.com/logo-dark.png" + assert result.values["favicon_url"] == "https://env.example.com/favicon.ico" + assert (result.logo_url_via_proxy, result.logo_url_dark_via_proxy) == (False, False) def test_get_ui_theme_settings_reports_unset_when_absent_everywhere( self, mock_proxy_config, monkeypatch @@ -1403,22 +1413,45 @@ class TestProxySettingEndpoints: assert values["logo_url"] is None assert values["favicon_url"] is None - def test_get_ui_theme_settings_does_not_disclose_local_path_env_value( - self, mock_proxy_config, monkeypatch - ): - """This endpoint is public, so an env-configured local filesystem branding - path must never be surfaced to anonymous callers; only public http(s) URLs. - """ - monkeypatch.setenv("UI_LOGO_PATH", "/mnt/secret/internal/logo.png") + @pytest.mark.usefixtures("mock_proxy_config") + @pytest.mark.parametrize("light,public_light,light_via_proxy", [ + (None, None, False), + (" ", None, False), + ("https://cdn.example.com/light.png", "https://cdn.example.com/light.png", False), + ("/mnt/secret/internal/light.png", None, True), + ]) + @pytest.mark.parametrize("dark,public_dark,dark_via_proxy", [ + (None, None, False), + (" ", None, False), + ("https://cdn.example.com/dark.png", "https://cdn.example.com/dark.png", False), + ("/mnt/secret/internal/dark.png", None, True), + ]) + def test_get_ui_theme_settings_separates_private_logo_rendering_from_settings( + self, monkeypatch: pytest.MonkeyPatch, + light: str | None, public_light: str | None, light_via_proxy: bool, + dark: str | None, public_dark: str | None, dark_via_proxy: bool, + ) -> None: + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import UIThemeSettingsResponse + + for key, value in (("UI_LOGO_PATH", light), ("UI_LOGO_PATH_DARK", dark)): + if value is None: + monkeypatch.delenv(key, raising=False) + else: + monkeypatch.setenv(key, value) monkeypatch.setenv("LITELLM_FAVICON_URL", "file:///etc/favicon.ico") - response = client.get("/get/ui_theme_settings") + response: Final = client.get("/get/ui_theme_settings") assert response.status_code == 200 - values = response.json()["values"] - # the local path / file scheme is withheld rather than disclosed - assert values["logo_url"] is None - assert values["favicon_url"] is None + result: Final = UIThemeSettingsResponse.model_validate(response.json()) + assert (result.values["logo_url"], result.values["logo_url_dark"]) == (public_light, public_dark) + assert (result.logo_url_via_proxy, result.logo_url_dark_via_proxy) == (light_via_proxy, dark_via_proxy) + assert result.values["favicon_url"] is None + assert "/mnt/secret/internal/" not in response.text + assert "file:///etc/favicon.ico" not in response.text + for hint in ("logo_url_via_proxy", "logo_url_dark_via_proxy"): + assert hint not in result.values + assert hint not in json.dumps(result.field_schema) def test_get_ui_settings(self, mock_auth, monkeypatch): """Test retrieving UI settings with allowlist sanitization""" diff --git a/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py b/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py index 50f50478ad3..8f8f10565e9 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py +++ b/tests/unit/proxy/utils/proxy_logging/test_streaming_hooks.py @@ -23,7 +23,7 @@ from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( +from litellm.llms.anthropic.messages.native.streaming_iterator import ( BaseAnthropicMessagesStreamingIterator, ) from litellm.proxy._types import UserAPIKeyAuth diff --git a/tests/unit/responses/mcp/test_tool_precheck.py b/tests/unit/responses/mcp/test_tool_precheck.py index 6bf7b6eddb7..bd37114ccea 100644 --- a/tests/unit/responses/mcp/test_tool_precheck.py +++ b/tests/unit/responses/mcp/test_tool_precheck.py @@ -18,7 +18,7 @@ GATEWAY_REGISTRY_MODULE: Final = "litellm.proxy.mcp_server.mcp_server_manager" GATEWAY_HANDLER_MODULES: Final = ( "litellm.responses.mcp.litellm_proxy_mcp_handler", "litellm.responses.mcp.chat_completions_handler", - "litellm.llms.anthropic.pass_through.messages.mcp_handler", + "litellm.llms.anthropic.messages.native.mcp_handler", ) FUNCTION_TOOL: Final = { diff --git a/tests/unit/router_strategy/test_router_tag_regex_routing.py b/tests/unit/router_strategy/test_router_tag_regex_routing.py index 212c2627d88..46e79a1b8c8 100644 --- a/tests/unit/router_strategy/test_router_tag_regex_routing.py +++ b/tests/unit/router_strategy/test_router_tag_regex_routing.py @@ -121,6 +121,7 @@ async def test_claude_code_ua_routes_to_cc_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "claude-code/1.2.3"}}, ) assert len(result) == 1 @@ -135,6 +136,7 @@ async def test_regular_ua_routes_to_default_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (browser)"}}, ) assert len(result) == 1 @@ -149,6 +151,7 @@ async def test_no_ua_routes_to_default_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {}}, ) assert len(result) == 1 @@ -164,6 +167,7 @@ async def test_tag_routing_metadata_written_for_regex_match(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": metadata}, ) assert "tag_routing" in metadata @@ -181,6 +185,7 @@ async def test_tag_filtering_disabled_returns_all_deployments(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}}, ) assert result == ALL_DEPLOYMENTS @@ -199,6 +204,7 @@ async def test_empty_healthy_deployments_with_request_tags_returns_empty_list(): llm_router_instance=router, model="gpt-5.2", healthy_deployments=[], + request_deployments=[], request_kwargs={ "metadata": {"tags": ["client_id:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]} }, @@ -228,6 +234,7 @@ async def test_explicit_tag_match_takes_precedence_over_regex(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_with_both], + request_deployments=[deployment_with_both], request_kwargs={"metadata": metadata}, ) assert len(result) == 1 @@ -269,6 +276,7 @@ async def test_user_agent_present_no_tag_regex_deployments_does_not_raise(): llm_router_instance=router, model="gpt-4", healthy_deployments=plain_tag_only_deployments, + request_deployments=plain_tag_only_deployments, request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (any-client)"}}, ) # Falls through to "return healthy_deployments" path unchanged @@ -305,6 +313,7 @@ async def test_tag_routing_metadata_not_overwritten_for_multiple_matches(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_a, deployment_b], + request_deployments=[deployment_a, deployment_b], request_kwargs={"metadata": metadata}, ) assert len(result) == 2 @@ -357,6 +366,7 @@ async def test_match_any_false_strict_tag_check_blocks_regex_fallback(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_strict, default_deployment], + request_deployments=[deployment_strict, default_deployment], request_kwargs={"metadata": metadata}, ) ids = [d["model_info"]["id"] for d in result] @@ -387,6 +397,7 @@ async def test_match_any_false_regex_only_deployment_still_matches(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[regex_only_deployment], + request_deployments=[regex_only_deployment], request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}}, ) assert len(result) == 1 diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index 097a34cf2e6..26d04be7e25 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -3,12 +3,14 @@ import logging -from typing import Final +from typing import Final, Literal +from unittest.mock import patch import pytest import litellm from litellm._logging import verbose_logger +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_strategy.tag_based_routing import get_deployments_for_tag from litellm.types.router import RouterErrors @@ -1367,14 +1369,17 @@ def test_chain_allows_fail_open_true_when_any_member_sets_flag(): {"model_info": {}, "litellm_params": {"tags": ["provider:anthropic"]}}, {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["provider:openai"]}}, ] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) is True + assert ( + _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) + is True + ) def test_chain_allows_fail_open_false_by_default(): from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open deployments = [{"model_info": {}}, {}] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset(), frozenset()) is False + assert _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset(), frozenset()) is False def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all(): @@ -1386,7 +1391,10 @@ def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all(): deployments = [ {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["default", "reasoning_type:low"]}}, ] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) is True + assert ( + _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) + is True + ) def test_unknown_required_tag_hides_an_answer_denies_fail_open(): @@ -1407,7 +1415,11 @@ def test_unknown_required_tag_hides_an_answer_denies_fail_open(): # fail-open must be denied even though the flag is set on the group. assert ( _chain_allows_fail_open( - deployments, frozenset(), frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), frozenset() + deployments, + deployments, + frozenset(), + frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), + frozenset(), ) is False ) @@ -1434,7 +1446,9 @@ def test_unknown_required_tag_allows_fail_open_when_no_answer_is_hidden(): # carries both, so this is a genuinely unsatisfiable combination, not an invented # tag masking a narrower answer. Fail-open must proceed normally. assert ( - _chain_allows_fail_open(deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset()) + _chain_allows_fail_open( + deployments, deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset() + ) is True ) @@ -1496,8 +1510,8 @@ def test_chain_allows_fail_open_true_when_prefixed_unknown_required_tag_is_confi }, ] required_set = frozenset({"provider:anthropic", "typo-tag"}) - assert _chain_allows_fail_open(deployments, frozenset(), required_set, frozenset()) is False - assert _chain_allows_fail_open(deployments, frozenset(), required_set, required_set) is True + assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, frozenset()) is False + assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, required_set) is True # --- get_deployments_for_tag required-AND ("&") integration tests --- @@ -1765,6 +1779,141 @@ async def test_required_and_combined_with_positive_unmatched_raises_by_default() # --- get_deployments_for_tag allow_fail_open integration tests --- +def _fail_open_router(opted_in_group: str | None = "chat", route: str = "model") -> litellm.Router: + model_name: Final = "chat-*" if route == "wildcard" else "chat" + team_info: Final = {"team_id": "team-a", "team_public_model_name": "public-chat"} if route == "team" else {} + return litellm.Router( + model_list=[ + { + "model_name": model_name if opted_in_group in (None, "chat") else opted_in_group, + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["vision"]}, + "model_info": {"id": "opted-in", "allow_fail_open": opted_in_group is not None, **team_info}, + }, + { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["region:eu"]}, + "model_info": {"id": "remaining", **team_info}, + }, + ], + routing_groups=( + [{"group_name": "public-chat", "models": ["chat"], "routing_strategy": "simple-shuffle"}] + if route == "routing_group" + else None + ), + model_group_alias={"public-chat": "chat"} if route == "alias" else {}, + enable_tag_filtering=True, + optional_pre_call_checks=["responses_api_deployment_check"], + ) + + +def _response_id_for_remaining_deployment() -> str: + return ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="remaining", response_id="resp_previous" + ) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("narrowing", ("affinity", "cooldown")) +@pytest.mark.parametrize("tags", (("component:chat-search",), ("&vision",), ("!region:eu",))) +@pytest.mark.parametrize("route", ("model", "routing_group", "wildcard", "team", "alias")) +async def test_tag_fail_open_survives_deployment_narrowing( + narrowing: Literal["affinity", "cooldown"], tags: tuple[str, ...], route: str +) -> None: + router: Final = _fail_open_router(route=route) + requested_model: Final = "chat" if route == "model" else "chat-test" if route == "wildcard" else "public-chat" + with patch("time.time", return_value=1_700_000_000.0): + if narrowing == "cooldown": + router.cooldown_cache.add_deployment_to_cooldown( + model_id="opted-in", + original_exception=RuntimeError("unavailable"), + exception_status=500, + cooldown_time=60, + ) + candidates: Final = await router.async_get_healthy_deployments( + model=requested_model, + request_kwargs={ + "litellm_metadata": {"tags": list(tags), "inherited_tags": [], "user_api_key_team_id": "team-a"}, + **( + {"previous_response_id": _response_id_for_remaining_deployment()} if narrowing == "affinity" else {} + ), + }, + ) + assert isinstance(candidates, (list, tuple)) + assert tuple(deployment["model_info"]["id"] for deployment in candidates) == ("remaining",) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("route", ("routing_group", "wildcard", "team", "alias")) +async def test_tag_fail_open_resolves_public_model_names(route: str) -> None: + router: Final = _fail_open_router(route=route) + candidates: Final = await router.async_get_healthy_deployments( + model="chat-test" if route == "wildcard" else "public-chat", + request_kwargs={ + "litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-a"}, + }, + ) + assert isinstance(candidates, (list, tuple)) + assert {deployment["model_info"]["id"] for deployment in candidates} == {"opted-in", "remaining"} + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("opted_in_group", (None, "other-group")) +async def test_tag_fail_open_requires_opt_in_in_the_requested_group(opted_in_group: str | None) -> None: + router: Final = _fail_open_router(opted_in_group) + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="chat", + request_kwargs={ + "previous_response_id": _response_id_for_remaining_deployment(), + "litellm_metadata": {"tags": ["component:chat-search"], "inherited_tags": []}, + }, + ) + + +@pytest.mark.asyncio() +async def test_tag_fail_open_does_not_use_another_teams_opt_in() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused"}, + "model_info": { + "id": team_id, + "team_id": team_id, + "team_public_model_name": "public-chat", + "allow_fail_open": team_id == "team-a", + }, + } + for team_id in ("team-a", "team-b") + ], + enable_tag_filtering=True, + ) + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="public-chat", + request_kwargs={ + "litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-b"}, + }, + ) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("inherited_tag", ("&vision", "!region:eu")) +async def test_tag_fail_open_preserves_inherited_policy_after_affinity(inherited_tag: str) -> None: + router: Final = _fail_open_router() + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="chat", + request_kwargs={ + "previous_response_id": _response_id_for_remaining_deployment(), + "litellm_metadata": { + "tags": [inherited_tag, "component:chat-search"], + "inherited_tags": [inherited_tag], + }, + }, + ) + + @pytest.mark.asyncio() async def test_allow_fail_open_required_and_unmatched_falls_back_to_default_pool(): router = litellm.Router( @@ -3305,10 +3454,12 @@ async def _routed_to(router: litellm.Router, metadata: dict[str, object]) -> str async def _pool(router: litellm.Router, metadata: dict[str, object]) -> frozenset[str]: + request_deployments: Final = router.get_model_list(model_name="chat") deployments: Final = await get_deployments_for_tag( llm_router_instance=router, model="chat", - healthy_deployments=router.get_model_list(model_name="chat"), + healthy_deployments=request_deployments, + request_deployments=request_deployments, request_kwargs={"metadata": metadata}, ) return frozenset(deployment["model_info"]["id"] for deployment in deployments) diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 75b6ab04446..53375ee0ab4 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -16,7 +16,7 @@ from pydantic import TypeAdapter import litellm from litellm import Router from litellm.integrations.custom_logger import CustomLogger -from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import FakeAnthropicMessagesStreamIterator +from litellm.llms.anthropic.messages.native.fake_stream_iterator import FakeAnthropicMessagesStreamIterator from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper diff --git a/tests/unit/rust_bridge/messages/test_secrets.py b/tests/unit/rust_bridge/messages/test_secrets.py index 64208182025..f6439753809 100644 --- a/tests/unit/rust_bridge/messages/test_secrets.py +++ b/tests/unit/rust_bridge/messages/test_secrets.py @@ -9,7 +9,7 @@ import pytest import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager -from litellm.llms.anthropic.pass_through.messages.handler import anthropic_messages +from litellm.llms.anthropic.messages.native.handler import anthropic_messages from litellm.rust_bridge.host import settings from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES from litellm.rust_bridge.public_call import NativeCall diff --git a/tests/unit/systemone/test_main.py b/tests/unit/systemone/test_main.py index 05a9850a221..80713c1ea2b 100644 --- a/tests/unit/systemone/test_main.py +++ b/tests/unit/systemone/test_main.py @@ -5,8 +5,10 @@ import json from collections.abc import Mapping from types import MappingProxyType from typing import Final +from uuid import uuid4 import pytest +import responses import respx from respx.models import Call @@ -34,6 +36,48 @@ _CLOUDFLARE_CLEF_URL: Final = f"{_CLOUDFLARE_API_BASE}/@cf/cloudflare/clef" _PNG_DATA_URL: Final = "data:image/png;base64,AA==" +@responses.activate +@pytest.mark.parametrize("use_async", (True, False), ids=("asystemone", "systemone")) +@pytest.mark.parametrize("model", ("ai_decide", "serving-decider")) +@pytest.mark.parametrize("base_from_env", (True, False)) +async def test_should_authenticate_databricks_systemone_with_oauth_m2m( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + use_async: bool, + model: str, + base_from_env: bool, +) -> None: + monkeypatch.setenv("DATABRICKS_CLIENT_ID", "synthetic-client") + monkeypatch.setenv("DATABRICKS_CLIENT_SECRET", f"synthetic-{uuid4().hex}") + monkeypatch.setenv("DATABRICKS_API_BASE", "https://workspace.example/serving-endpoints") + token: Final = responses.post( + "https://workspace.example/oidc/v1/token", json={"access_token": "oauth-token", "expires_in": 3600} + ) + url: Final = ( + "https://workspace.example/api/2.0/ai-functions/ai-decide" + if model == "ai_decide" + else f"https://workspace.example/serving-endpoints/{model}/invocations" + ) + upstream_response: Final = {"answers": {"is_refund": {"type": "noul", "noul": 0.93}}} + route: Final = respx_mock.post(url).respond( + json={"response": upstream_response} if model == "ai_decide" else upstream_response + ) + kwargs: Final[Mapping[str, object]] = { + "model": f"databricks/{model}", + "state": _STATE, + "questions": _SYSTEMONE_QUESTIONS, + **({} if base_from_env else {"api_base": "https://workspace.example/serving-endpoints"}), + } + + for _ in range(2): + result: Final = await litellm.asystemone(**kwargs) if use_async else litellm.systemone(**kwargs) + assert isinstance(result, DecisionsResponse) + assert result.answers["is_refund"].model_dump(mode="json") == {"type": "noul", "noul": 0.93} + assert route.calls.last.request.headers["Authorization"] == "Bearer oauth-token" + assert route.call_count == 2 + assert token.call_count == 1 + + @pytest.fixture(autouse=True) def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) diff --git a/tests/unit/test_internal_context.py b/tests/unit/test_internal_context.py index 38b04138f9f..f78c6064f76 100644 --- a/tests/unit/test_internal_context.py +++ b/tests/unit/test_internal_context.py @@ -54,7 +54,7 @@ _IN_MEMORY_ONLY_CALLERS: Final = frozenset( "litellm/llms/azure/common_utils.py", "litellm/llms/bedrock/base_aws_llm.py", "litellm/llms/custom_httpx/http_handler.py", - "litellm/llms/databricks/agent/transformation.py", + "litellm/llms/databricks/common_utils.py", "litellm/llms/gigachat/authenticator.py", "litellm/llms/github_copilot/per_user_auth.py", "litellm/llms/litellm_proxy/skills/handler.py", diff --git a/tests/unit/test_private_usage_aliases.py b/tests/unit/test_private_usage_aliases.py index 924d1f1fc2b..1d9c2e87100 100644 --- a/tests/unit/test_private_usage_aliases.py +++ b/tests/unit/test_private_usage_aliases.py @@ -2108,49 +2108,49 @@ PROPERTY_CASES: Final = ( CLASS_PROPERTY_CASES: Final = () LLMS_ALIAS_CASES: Final = ( ( - "litellm.llms.anthropic.pass_through.adapters.transformation", + "litellm.llms.anthropic.messages.adapters.transformation", "LiteLLMAnthropicMessagesAdapter", "_translate_openai_usage_to_anthropic_usage", "translate_openai_usage_to_anthropic_usage", False, ), ( - "litellm.llms.anthropic.pass_through.adapters.transformation", + "litellm.llms.anthropic.messages.adapters.transformation", "LiteLLMAnthropicMessagesAdapter", "_translate_openai_usage_to_anthropic_usage_delta", "translate_openai_usage_to_anthropic_usage_delta", False, ), ( - "litellm.llms.anthropic.pass_through.adapters.transformation", + "litellm.llms.anthropic.messages.adapters.transformation", "LiteLLMAnthropicMessagesAdapter", "_translate_streaming_openai_chunk_to_anthropic_content_block", "translate_streaming_openai_chunk_to_anthropic_content_block", False, ), ( - "litellm.llms.anthropic.pass_through.context_management.dispatcher", + "litellm.llms.anthropic.messages.context_management.dispatcher", "", "_normalize_spec", "normalize_spec", False, ), ( - "litellm.llms.anthropic.pass_through.messages.streaming_iterator", + "litellm.llms.anthropic.messages.native.streaming_iterator", "", "_is_message_stop_chunk", "is_message_stop_chunk", False, ), ( - "litellm.llms.anthropic.pass_through.messages.streaming_iterator", + "litellm.llms.anthropic.messages.native.streaming_iterator", "", "_is_provider_error_chunk", "is_provider_error_chunk", False, ), ( - "litellm.llms.anthropic.pass_through.messages.transformation", + "litellm.llms.anthropic.messages.native.transformation", "", "_messages_carry_output_config", "messages_carry_output_config", diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3004f83b1df..e2ee80b7874 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -39,8 +39,8 @@ from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from litellm.llms.anthropic.pass_through.messages.streaming_iterator import closing_superseded_attempt -from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import ( +from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt +from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, ) from litellm.llms.bedrock.common_utils import BedrockError diff --git a/ui/litellm-dashboard/src/app/(dashboard)/ui-theme/UIThemeSettings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/ui-theme/UIThemeSettings.test.tsx index 4b58590ffd1..86ef33c0d8b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/ui-theme/UIThemeSettings.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/ui-theme/UIThemeSettings.test.tsx @@ -10,7 +10,8 @@ const setLogoUrl = vi.fn(); const setLogoUrlDark = vi.fn(); const setFaviconUrl = vi.fn(); -vi.mock("@/contexts/ThemeContext", () => ({ +vi.mock("@/contexts/ThemeContext", async (importOriginal) => ({ + ...(await importOriginal()), useTheme: () => ({ logoUrl: null, setLogoUrl, @@ -22,7 +23,7 @@ vi.mock("@/contexts/ThemeContext", () => ({ })); vi.mock("@/components/networking", () => ({ - getProxyBaseUrl: () => "", + getProxyBaseUrl: () => "/gateway", getGlobalLitellmHeaderName: () => "Authorization", })); @@ -111,6 +112,34 @@ describe("UIThemeSettings", () => { expect(bodyOf(patchCalls()[0]).logo_url_dark).toBe("https://a.test/logo-dark.png"); }); + it.each(["Save Changes", "Reset to Default"])( + "should keep local logo routes out of settings and clear preview overrides with %s", + async (action) => { + const user = userEvent.setup(); + fetchMock.mockImplementation(async () => + Response.json({ + values: { logo_url: null, logo_url_dark: null }, + logo_url_via_proxy: true, + logo_url_dark_via_proxy: true, + }), + ); + + render(); + + await waitFor(() => expect(setLogoUrlDark).toHaveBeenCalledWith("/gateway/get_image?theme=dark")); + expect(setLogoUrl).toHaveBeenCalledWith("/gateway/get_image"); + expect(screen.getByPlaceholderText(LOGO_PLACEHOLDER)).toHaveValue(""); + expect(screen.getByPlaceholderText(LOGO_DARK_PLACEHOLDER)).toHaveValue(""); + + await user.click(screen.getByRole("button", { name: action })); + await waitFor(() => expect(patchCalls()).toHaveLength(1)); + expect(bodyOf(patchCalls()[0])).toEqual({ logo_url: null, logo_url_dark: null, favicon_url: null }); + + expect(setLogoUrl).toHaveBeenLastCalledWith(null); + expect(setLogoUrlDark).toHaveBeenLastCalledWith(null); + }, + ); + it("should surface a backend failure when saving fails", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/ui-theme/UIThemeSettings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/ui-theme/UIThemeSettings.tsx index b6bc0f28dd5..c0b3bd26001 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/ui-theme/UIThemeSettings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/ui-theme/UIThemeSettings.tsx @@ -4,7 +4,7 @@ import { Card, CardContent } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; -import { useTheme } from "@/contexts/ThemeContext"; +import { resolveThemeImages, useTheme } from "@/contexts/ThemeContext"; import { getProxyBaseUrl, getGlobalLitellmHeaderName } from "@/components/networking"; import { toast } from "@/lib/toast"; @@ -40,12 +40,13 @@ const UIThemeSettings: React.FC = ({ userID, userRole, acc }); if (response.ok) { const data = await response.json(); + const images = resolveThemeImages(data, proxyBaseUrl); setLogoUrlInput(data.values?.logo_url || ""); setLogoUrlDarkInput(data.values?.logo_url_dark || ""); setFaviconUrlInput(data.values?.favicon_url || ""); - setLogoUrl(data.values?.logo_url || null); - setLogoUrlDark(data.values?.logo_url_dark || null); - setFaviconUrl(data.values?.favicon_url || null); + setLogoUrl(images.logoUrl); + setLogoUrlDark(images.logoUrlDark); + setFaviconUrl(images.faviconUrl); } } catch (error) { console.error("Error fetching theme settings:", error); diff --git a/ui/litellm-dashboard/src/app/chat/layout.integration.test.tsx b/ui/litellm-dashboard/src/app/chat/layout.integration.test.tsx index a7189ea6cd7..2efe698ec69 100644 --- a/ui/litellm-dashboard/src/app/chat/layout.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/chat/layout.integration.test.tsx @@ -1,7 +1,8 @@ import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders, screen, testQueryClient } from "@/../tests/test-utils"; +import { renderWithProviders, screen, testQueryClient, within } from "@/../tests/test-utils"; import { setServerRootPath } from "@/lib/serverRootPath"; +import { getProxyBaseUrl } from "@/components/networking"; import ChatLayout from "./layout"; const { router, route } = vi.hoisted(() => ({ @@ -33,9 +34,13 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/useUISettings", () => ({ const plugin = { name: "observability", display_name: "Observability", url: "https://plugin.example.com" }; const originalLocation = Object.getOwnPropertyDescriptor(window, "location")!; const assign = vi.fn(); +const LIGHT_LOGO = "https://cdn.test/light.svg"; +const DARK_LOGO = "https://cdn.test/dark.svg"; +let themeResponse: unknown = { values: {} }; describe("Chat layout product navigation", () => { beforeEach(() => { + themeResponse = { values: {} }; route.pathname = "/ui/chat"; localStorage.clear(); localStorage.setItem("litellm_plugin_mode", plugin.name); @@ -55,7 +60,7 @@ describe("Chat layout product navigation", () => { expect(new Headers(init?.headers).get("Authorization")).toBe("Bearer chat-layout-test-token"); return Response.json([plugin]); case "/get/ui_theme_settings": - return Response.json({ values: {} }); + return Response.json(themeResponse); case "/sso/get/ui_settings": return Response.json({}); default: @@ -75,6 +80,55 @@ describe("Chat layout product navigation", () => { vi.clearAllMocks(); }); + it.each([ + ["public", [LIGHT_LOGO, DARK_LOGO], [false, false], [LIGHT_LOGO, DARK_LOGO]], + ["local", [null, null], [true, true], ["/get_image", "/get_image?theme=dark"]], + ["public light and local dark", [LIGHT_LOGO, null], [false, true], [LIGHT_LOGO, "/get_image?theme=dark"]], + ["local light and public dark", [null, DARK_LOGO], [true, false], ["/get_image", DARK_LOGO]], + ["public light only", [LIGHT_LOGO, null], [false, false], [LIGHT_LOGO, LIGHT_LOGO]], + ["public dark only", [null, DARK_LOGO], [false, false], [DARK_LOGO, DARK_LOGO]], + ["local light only", [null, null], [true, false], ["/get_image", "/get_image"]], + ["local dark only", [null, null], [false, true], ["/get_image?theme=dark", "/get_image?theme=dark"]], + ] as const)( + "should render %s organization logos from the theme response", + async ( + _label, + [logo_url, logo_url_dark], + [logo_url_via_proxy, logo_url_dark_via_proxy], + [lightSource, darkSource], + ) => { + themeResponse = { values: { logo_url, logo_url_dark }, logo_url_via_proxy, logo_url_dark_via_proxy }; + const user = userEvent.setup(); + renderWithProviders( + +

Chat content

+ , + ); + + await user.click(screen.getByRole("button", { name: "Open chat navigation" })); + const logo = await screen.findByRole("img", { name: "Organization logo" }); + const [light, dark] = within(logo).getAllByAltText(""); + const source = (path: string) => (path.startsWith("/") ? `${getProxyBaseUrl()}${path}` : path); + expect(light).toHaveAttribute("src", source(lightSource)); + expect(dark).toHaveAttribute("src", source(darkSource)); + expect(light).toHaveClass("dark:hidden"); + expect(dark).toHaveClass("hidden", "dark:block"); + }, + ); + + it("should omit organization branding when no logo is configured", async () => { + const user = userEvent.setup(); + renderWithProviders( + +

Chat content

+ , + ); + + await user.click(screen.getByRole("button", { name: "Open chat navigation" })); + expect(await screen.findByRole("dialog", { name: "LiteLLM Chat" })).toBeInTheDocument(); + expect(screen.queryByRole("img", { name: "Organization logo" })).not.toBeInTheDocument(); + }); + it.each([ { pathname: "/ui/chat", root: "/", gateway: "/ui/" }, { pathname: "/gateway/ui/chat/models/", root: "/gateway", gateway: "/gateway/ui/" }, diff --git a/ui/litellm-dashboard/src/components/chat/ChatShell.integration.test.tsx b/ui/litellm-dashboard/src/components/chat/ChatShell.integration.test.tsx index c1af91845ee..3c7b9ef1b1a 100644 --- a/ui/litellm-dashboard/src/components/chat/ChatShell.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/chat/ChatShell.integration.test.tsx @@ -1,9 +1,13 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { renderWithProviders as render } from "@/../tests/test-utils"; +import { renderWithProviders } from "@/../tests/test-utils"; +import type { ReactElement } from "react"; +import { ThemeProvider } from "@/contexts/ThemeContext"; import ChatShell from "./ChatShell"; +const render = (ui: ReactElement) => renderWithProviders({ui}); + const { mockPush, mockUsePathname, mockUseChatShell, mockUseMediaQuery } = vi.hoisted(() => ({ mockPush: vi.fn(), mockUsePathname: vi.fn(() => "/ui/chat"), @@ -34,7 +38,15 @@ vi.mock("usehooks-ts", () => ({ useMediaQuery: mockUseMediaQuery })); vi.mock("./ConversationList", () => ({ default: () =>
})); describe("ChatShell", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ values: {} })), + ); + }); + afterEach(() => { + vi.unstubAllGlobals(); mockPush.mockClear(); mockUsePathname.mockReturnValue("/ui/chat"); mockUseMediaQuery.mockReturnValue(true); diff --git a/ui/litellm-dashboard/src/components/chat/ChatShell.tsx b/ui/litellm-dashboard/src/components/chat/ChatShell.tsx index c089a709b64..280e2d10b8c 100644 --- a/ui/litellm-dashboard/src/components/chat/ChatShell.tsx +++ b/ui/litellm-dashboard/src/components/chat/ChatShell.tsx @@ -1,6 +1,7 @@ "use client"; import React, { useState } from "react"; +import Image from "next/image"; import { usePathname, useRouter } from "next/navigation"; import { Plus, @@ -34,6 +35,7 @@ import ThemeToggle from "@/components/ThemeToggle/ThemeToggle"; import { useLogout } from "@/app/(dashboard)/hooks/useLogout"; import { uiHref } from "@/utils/uiHref"; import { useChatShell } from "@/contexts/ChatShellContext"; +import { useTheme } from "@/contexts/ThemeContext"; import ViewSwitcher from "@/components/Navbar/ViewSwitcher"; import ConversationList from "./ConversationList"; @@ -94,6 +96,9 @@ const ChatShell: React.FC<{ children: React.ReactNode }> = ({ children }) => { const [searchOpen, setSearchOpen] = useState(false); if (isDesktop && sidebarOpen) setSidebarOpen(false); const collapsed = isDesktop && sidebarCollapsed; + const { logoUrl, logoUrlDark } = useTheme(); + const lightLogo = logoUrl || logoUrlDark; + const darkLogo = logoUrlDark || logoUrl; const routes = getChatRoutes(); const isChatsRoute = pathname === routes.chats; const accountName = userEmail?.split("@")[0] || userId || "Account"; @@ -216,6 +221,26 @@ const ChatShell: React.FC<{ children: React.ReactNode }> = ({ children }) => { hideSearchButton />
+ {lightLogo && darkLogo && ( +
+ + +
+ )}