scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-05 07:37:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 233

merge: origin/main into litellm_rename_mcp_drop_experimental

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-10-27 17:10:44 +00:00
commit 99e471debd
221 changed files with 2520 additions and 721 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
ARCHITECTURE.md
View file

@ -336,9 +336,9 @@ Each translation is isolated in its own file, making it easy to test and modify
| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/pass_through/messages/transformation.py` |
| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/messages/native/transformation.py` |
| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py` |
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py` |
| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
### Example: Debugging prompt caching

2
litellm-rust/crates/callbacks-legacy-python/src/mapping.rs
View file

@ -235,7 +235,7 @@ python_callbacks! {
"async_log_stream_event",
"async_post_mcp_tool_call_hook",
],
Dispatch::Python("litellm.llms.anthropic.pass_through.messages.handler") => [
Dispatch::Python("litellm.llms.anthropic.messages.native.handler") => [
"async_pre_request_hook",
],
Dispatch::Python("litellm.litellm_core_utils.streaming_handler") => [

8
litellm/__init__.py
View file

@ -1729,6 +1729,9 @@ if TYPE_CHECKING:
from .llms.hosted_vllm.decisions.transformation import (
HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig,
)
from .llms.lm_studio.decisions.transformation import (
LMStudioDecisionsConfig as LMStudioDecisionsConfig,
)
from .llms.laya.decisions.transformation import (
LayaDecisionsConfig as LayaDecisionsConfig,
)
@ -1738,6 +1741,9 @@ if TYPE_CHECKING:
from .llms.openai.decisions.transformation import (
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
)
from .llms.azure.decisions.transformation import (
AzureDecisionsConfig as AzureDecisionsConfig,
)
from .llms.azure_ai.decisions.transformation import (
AzureAIDecisionsConfig as AzureAIDecisionsConfig,
)
@ -1782,7 +1788,7 @@ if TYPE_CHECKING:
SagemakerNovaConfig as SagemakerNovaConfig,
)
from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig
from .llms.anthropic.pass_through.messages.transformation import (
from .llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig as AnthropicMessagesConfig,
)
from .llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import (

6
litellm/_lazy_imports_registry.py
View file

@ -170,9 +170,11 @@ LLM_CONFIG_NAMES: Final = (
"StrandsDeciderDecisionsConfig",
"DatabricksDecisionsConfig",
"HostedVLLMDecisionsConfig",
"LMStudioDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"OpenAIDecisionsConfig",
"AzureDecisionsConfig",
"AzureAIDecisionsConfig",
"NvidiaNimRerankConfig",
"NvidiaNimRankingConfig",
@ -746,9 +748,11 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
".llms.hosted_vllm.decisions.transformation",
"HostedVLLMDecisionsConfig",
),
"LMStudioDecisionsConfig": (".llms.lm_studio.decisions.transformation", "LMStudioDecisionsConfig"),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
"NvidiaNimRerankConfig": (
".llms.nvidia_nim.rerank.transformation",
@ -797,7 +801,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
),
"CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"),
"AnthropicMessagesConfig": (
".llms.anthropic.pass_through.messages.transformation",
".llms.anthropic.messages.native.transformation",
"AnthropicMessagesConfig",
),
"BedrockClaudePlatformMessagesConfig": (

8
litellm/caching/caching_handler.py
View file

@ -56,7 +56,7 @@ from litellm.types.utils import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.pass_through.messages.response_cache import (
from litellm.llms.anthropic.messages.native.response_cache import (
AnthropicMessagesStreamCacheWriter,
)
from litellm.types.utils import PromptTokensDetailsWrapper
@ -182,7 +182,7 @@ def _should_defer_streaming_cache_hit_callbacks(*, cached_result: object) -> boo
spend and callback records. A plain (non-stream) replay logs here, since nothing
else will.
"""
from litellm.llms.anthropic.pass_through.messages.response_cache import (
from litellm.llms.anthropic.messages.native.response_cache import (
CachedAnthropicMessagesStreamIterator,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
@ -1040,7 +1040,7 @@ class LLMCachingHandler:
elif (
call_type == CallTypes.anthropic_messages.value or call_type == CallTypes.aanthropic_messages.value
) and isinstance(cached_result, dict):
from litellm.llms.anthropic.pass_through.messages.response_cache import (
from litellm.llms.anthropic.messages.native.response_cache import (
convert_cached_anthropic_messages_result,
)
@ -1270,7 +1270,7 @@ class LLMCachingHandler:
return result
if not isinstance(result, AsyncIterator):
return result
from litellm.llms.anthropic.pass_through.messages.response_cache import (
from litellm.llms.anthropic.messages.native.response_cache import (
AnthropicMessagesStreamCacheWriter,
)

74
litellm/images/main.py
View file

@ -1,7 +1,7 @@
import asyncio
import contextvars
import importlib
from collections.abc import Coroutine
from collections.abc import Coroutine, Mapping
from functools import partial
from typing import TYPE_CHECKING, Final, Literal, cast, overload
@ -85,6 +85,47 @@ def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]:
return _ImageEditRequestUtils_cache
def _chatgpt_image_params(kwargs: Mapping[str, object]) -> GenericLiteLLMParams:
from litellm.responses.utils import ResponsesAPIRequestUtils
headers: Final = ResponsesAPIRequestUtils.merge_client_forwarded_headers(
TypeAdapter[dict[str, object] | None](dict[str, object] | None).validate_python(kwargs.get("extra_headers")),
TypeAdapter[dict[str, str] | None](dict[str, str] | None).validate_python(kwargs.get("headers")),
)
return GenericLiteLLMParams.model_validate({**kwargs, "extra_headers": headers})
def _chatgpt_image_headers(model: str, params: GenericLiteLLMParams, headers: Mapping[str, str]) -> Mapping[str, str]:
from litellm.llms.chatgpt.common_utils import get_chatgpt_request_credential
from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig
credential: Final = get_chatgpt_request_credential(params)
credential_headers: Final = (
{
"Authorization": f"Bearer {credential.access_token}",
**({"ChatGPT-Account-Id": credential.account_id} if credential.account_id else {}),
}
if credential is not None
else {}
)
explicit_account_id: Final = next(
(value for name, value in headers.items() if name.lower() == "chatgpt-account-id"), None
)
auth_headers: Final = ChatGPTResponsesAPIConfig().validate_environment(
headers=credential_headers, model=model, litellm_params=params
)
return {
**{k: v for k, v in auth_headers.items() if k.lower() not in ("content-type", "accept")},
**{
k: v
for k, v in headers.items()
if k.lower() not in ("authorization", "chatgpt-account-id", "cookie", "x-litellm-api-key")
},
**({"ChatGPT-Account-Id": explicit_account_id} if credential is None and explicit_account_id else {}),
"Accept": "application/json",
}
##### Image Generation #######################
@client
async def aimage_generation(*args, **kwargs) -> ImageResponse:
@ -121,6 +162,12 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse:
model=model,
custom_llm_provider=provider_hints.get("custom_llm_provider"),
api_base=provider_hints.get("api_base"),
litellm_params=(
_chatgpt_image_params(TypeAdapter(dict[str, object]).validate_python(kwargs))
if provider_hints.get("custom_llm_provider") == "chatgpt"
or (isinstance(model, str) and model.startswith("chatgpt/"))
else None
),
)
# Await normally
@ -240,12 +287,18 @@ def image_generation(
if extra_headers is not None:
headers.update(extra_headers)
model_response: ImageResponse = litellm.utils.ImageResponse()
chatgpt_params: Final = (
_chatgpt_image_params(TypeAdapter(dict[str, object]).validate_python({**kwargs, "api_key": api_key}))
if custom_llm_provider == "chatgpt" or (model is not None and model.startswith("chatgpt/"))
else None
)
dynamic_api_key: str | None = None
if model is not None or custom_llm_provider is not None:
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
litellm_params=chatgpt_params,
)
else:
model = "dall-e-2"
@ -268,7 +321,10 @@ def image_generation(
"size",
"style",
]
non_default_params: Final = filter_out_litellm_params(kwargs, excluding=openai_params)
non_default_params: Final = filter_out_litellm_params(
kwargs,
excluding=[*openai_params, "extra_headers", "headers"] if chatgpt_params is not None else openai_params,
)
image_generation_config: BaseImageGenerationConfig | None = None
if custom_llm_provider is not None and custom_llm_provider in LlmProviders._member_map_.values():
@ -490,7 +546,7 @@ def image_generation(
or custom_llm_provider == LlmProviders.LITELLM_PROXY.value
or custom_llm_provider in litellm.openai_compatible_providers
):
if extra_headers is not None:
if extra_headers is not None and chatgpt_params is None:
optional_params["extra_headers"] = extra_headers
# Forward OpenAI organization if present (set by proxy pre-call utils)
organization: Final[str | None] = kwargs.get("organization", None)
@ -506,7 +562,17 @@ def image_generation(
organization=organization,
aimg_generation=aimg_generation,
client=client,
headers=headers,
headers=(
dict(
_chatgpt_image_headers(
model,
chatgpt_params.model_copy(update={"api_key": dynamic_api_key or api_key}),
TypeAdapter(dict[str, str]).validate_python(headers),
)
)
if chatgpt_params is not None
else headers
),
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "bedrock":

2
litellm/integrations/otel/model/request_io.py
View file

@ -6,7 +6,7 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.integrations.otel.mappers.utils import json_or_none
from litellm.llms.anthropic.pass_through.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream
from litellm.llms.anthropic.messages.stream_assembly import assemble_anthropic_sse_stream, is_raw_sse_stream
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse
from litellm.types.utils import ModelResponse, ModelResponseStream

2
litellm/integrations/shadow_eval_logger.py
View file

@ -113,7 +113,7 @@ def _chat_request_from_anthropic_messages(
because the logged optional_params switch dialect per provider path (the bridge's
inner completion rewrites them to chat shape mid-flight); the adapter translates
them alongside the messages, and sampling params copy through untranslated."""
from litellm.llms.anthropic.pass_through.adapters.transformation import (
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)

2
litellm/integrations/websearch_interception/ARCHITECTURE.md
View file

@ -70,7 +70,7 @@ Claude Code (Anthropic's official CLI) sends web search requests using Anthropic
Native tools are converted to LiteLLM standard format **before** sending to the provider:
1. **Conversion Point** (`litellm/llms/anthropic/pass_through/messages/handler.py`):
1. **Conversion Point** (`litellm/llms/anthropic/messages/native/handler.py`):
- In `anthropic_messages()` function (lines 60-127)
- Runs BEFORE the API request is made
- Detects native web search tools using `is_web_search_tool()`

2
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -6354,7 +6354,7 @@ def is_flat_responses_tool(tool: Mapping[str, object]) -> bool:
def chat_completion_tools_for(tool: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)

87
litellm/litellm_core_utils/renamed_package_shim.py Normal file
View file

@ -0,0 +1,87 @@
import importlib
import sys
import warnings
from collections.abc import Callable, Sequence
from importlib.abc import Loader, MetaPathFinder
from importlib.machinery import ModuleSpec
from importlib.util import spec_from_loader
from types import ModuleType
from typing import Final
class _AliasLoader(Loader):
"""Hand the import system the module already loaded under its new name instead of executing a copy."""
def __init__(self, target: ModuleType) -> None:
self._target: Final = target
self._spec: Final = target.__spec__
def create_module(self, spec: ModuleSpec) -> ModuleType:
return self._target
def exec_module(self, module: ModuleType) -> None:
module.__spec__ = self._spec # rebind-ok: the loader restores the spec the import system replaced
class _RenamedPackageFinder(MetaPathFinder):
def __init__(self, old_name: str, renames: tuple[tuple[str, str], ...]) -> None:
self._prefix: Final = old_name + "."
self._renames: Final = renames
def new_name_for(self, old_module_name: str) -> str | None:
if not old_module_name.startswith(self._prefix):
return None
for old_prefix, new_prefix in self._renames:
if old_module_name == old_prefix or old_module_name.startswith(old_prefix + "."):
return new_prefix + old_module_name[len(old_prefix) :]
return None
def find_spec(
self,
fullname: str,
path: Sequence[str] | None = None,
target: ModuleType | None = None,
) -> ModuleSpec | None:
new_name: Final = self.new_name_for(fullname)
if new_name is None:
return None
module: Final = importlib.import_module(new_name)
spec: Final = spec_from_loader(fullname, _AliasLoader(module), is_package=hasattr(module, "__path__"))
if spec is not None and spec.submodule_search_locations is not None:
spec.submodule_search_locations = list(module.__path__) # pyright: ignore[reportAny] # module attributes are untyped
return spec
def alias_renamed_package(
old_name: str,
new_name: str,
*,
nested_renames: tuple[tuple[str, str], ...] = (),
) -> Callable[[str], ModuleType]:
"""Make every module of the package now at ``new_name`` importable under its old dotted name.
Nothing is imported up front: a meta path finder resolves ``old_name.x.y`` to the module object at
``new_name.x.y`` the moment something imports it, so a deep import of the old path is the very same object
as the new path and importing one old module loads no more than importing its new counterpart would.
``nested_renames`` lists (old, new) dotted prefixes for sub-packages that changed name inside the move,
longest first. The returned function is the shim package's module ``__getattr__``, so attribute access on
the old package reaches the renamed sub-packages too.
"""
warnings.warn(
f"{old_name} moved to {new_name}; update imports, the old path will be removed in a future release",
DeprecationWarning,
stacklevel=3,
)
finder: Final = _RenamedPackageFinder(old_name, (*nested_renames, (old_name, new_name)))
if not any(
isinstance(entry, _RenamedPackageFinder) and entry.new_name_for(old_name + ".x") for entry in sys.meta_path
):
sys.meta_path.insert(0, finder)
def getattr_(name: str) -> ModuleType:
try:
return importlib.import_module(f"{old_name}.{name}")
except ModuleNotFoundError as error:
raise AttributeError(f"module {old_name!r} has no attribute {name!r}") from error
return getattr_

2
litellm/litellm_core_utils/served_output_texts.py
View file

@ -110,7 +110,7 @@ def served_output_texts(response: object) -> ServedTexts | None:
def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None:
if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks):
return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream)))
from litellm.llms.anthropic.pass_through.stream_assembly import (
from litellm.llms.anthropic.messages.stream_assembly import (
assemble_anthropic_sse_stream,
is_anthropic_sse_stream,
)

61
litellm/litellm_core_utils/token_counter.py
View file

@ -261,6 +261,10 @@ def get_image_dimensions(
_header, encoded = data.split(",", 1)
img_data = base64.b64decode(encoded)
return _image_dimensions_from_bytes(img_data)
def _image_dimensions_from_bytes(img_data: bytes) -> tuple[int, int]:
img_type: Final = get_image_type(img_data)
if img_type == "png":
@ -307,8 +311,37 @@ def get_image_dimensions(
return DEFAULT_IMAGE_WIDTH, DEFAULT_IMAGE_HEIGHT
def _auto_image_dimensions(data: str) -> tuple[int, int] | None:
"""Read bounded inline image data without fetching caller-supplied URLs."""
if not data.startswith("data:image/"):
return None
max_bytes: Final = int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024)
max_encoded_length: Final = 4 * ((max_bytes + 2) // 3)
# Bound allocations before splitting or decoding the data URL, including its header.
if max_bytes <= 0 or len(data) > max_encoded_length + 128:
return None
header, separator, encoded = data.partition(",")
if not separator or len(header) > 128 or not header.endswith(";base64") or len(encoded) > max_encoded_length:
return None
try:
image_data: Final = base64.b64decode(encoded, validate=True)
if len(image_data) > max_bytes or get_image_type(image_data) not in (
"png",
"gif",
"jpeg",
"webp",
):
return None
width, height = _image_dimensions_from_bytes(image_data)
except (ValueError, TypeError, struct.error):
return None
if width <= 0 or height <= 0:
return None
return width, height
def calculate_img_tokens(
data,
data: str,
mode: Literal["low", "high", "auto"] = "auto",
base_tokens: int = 85, # openai default - https://openai.com/pricing
use_default_image_token_count: bool = False,
@ -318,7 +351,8 @@ def calculate_img_tokens(
Args:
data (str): The URL or base64 encoded string of the image.
mode (Literal["low", "high", "auto"]): The mode to use for calculating the number of tokens.
mode (Literal["low", "high", "auto"]): Auto uses high-detail tiles for readable inline images;
URLs and unreadable images keep the base estimate without network I/O.
base_tokens (int): The base number of tokens for an image.
use_default_image_token_count (bool): When True, will NOT make a GET request to the image URL and instead return the default image dimensions.
@ -328,18 +362,19 @@ def calculate_img_tokens(
if use_default_image_token_count:
verbose_logger.debug("Using default image token count: %s", DEFAULT_IMAGE_TOKEN_COUNT)
return DEFAULT_IMAGE_TOKEN_COUNT
if mode == "low" or mode == "auto":
if mode == "low":
return base_tokens
elif mode == "high":
# Run the async function using the helper
width, height = get_image_dimensions(
data=data,
)
resized_width, resized_height = resize_image_high_res(width=width, height=height)
tiles_needed_high_res = calculate_tiles_needed(resized_width=resized_width, resized_height=resized_height)
tile_tokens: Final = (base_tokens * 2) * tiles_needed_high_res
total_tokens: Final = base_tokens + tile_tokens
return total_tokens
if mode not in ("auto", "high"):
return None
dimensions: Final = _auto_image_dimensions(data) if mode == "auto" else get_image_dimensions(data=data)
if dimensions is None:
return base_tokens
width, height = dimensions
resized_width, resized_height = resize_image_high_res(width=width, height=height)
tiles_needed_high_res = calculate_tiles_needed(resized_width=resized_width, resized_height=resized_height)
tile_tokens: Final = (base_tokens * 2) * tiles_needed_high_res
total_tokens: Final = base_tokens + tile_tokens
return total_tokens
TokenCounterFunction = Callable[[str], int]

10
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -26,18 +26,18 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.pass_through.adapters.transformation import (
from litellm.llms.anthropic.messages.adapters.transformation import (
OPENAI_MAX_TOOL_NAME_LENGTH,
LiteLLMAnthropicMessagesAdapter,
is_provider_native_tool_dict,
truncate_tool_name,
)
from litellm.llms.anthropic.pass_through.convertible_blocks import (
from litellm.llms.anthropic.messages.convertible_blocks import (
keep_non_string_tool_result_texts,
scannable_tool_result_request,
)
from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.pass_through.stream_assembly import (
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.messages.stream_assembly import (
build_complete_streaming_response,
build_usage_only_response_from_chunks,
)
@ -435,7 +435,7 @@ class AnthropicMessagesHandler(BaseTranslation):
def _standalone_block_chunks(self, exc: "ModifyResponseException") -> list[bytes]:
import uuid
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.guardrail_translation.utils import (

2
litellm/llms/anthropic/count_tokens/transformation.py
View file

@ -162,7 +162,7 @@ def _has_non_chat_user_block(message: Mapping[str, object]) -> bool:
def _gets_placeholder_name(tool: Mapping[str, object]) -> bool:
from litellm.llms.anthropic.pass_through.adapters.transformation import is_provider_native_tool_dict
from litellm.llms.anthropic.messages.adapters.transformation import is_provider_native_tool_dict
name: Final = tool.get("name")
return not (_is_chat_tool(tool) or is_provider_native_tool_dict(tool) or (isinstance(name, str) and name.strip()))

0
litellm/llms/anthropic/pass_through/adapters/__init__.py → litellm/llms/anthropic/messages/adapters/__init__.py
View file

20
litellm/llms/anthropic/pass_through/adapters/handler.py → litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -15,23 +15,23 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.anthropic.pass_through.adapters.transformation import (
from litellm.llms.anthropic.messages.adapters.transformation import (
AnthropicAdapter,
)
from litellm.llms.anthropic.pass_through.context_management import (
from litellm.llms.anthropic.messages.context_management import (
AnthropicContextManagementError,
PolyfillResult,
apply_context_management,
)
from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text
from litellm.llms.anthropic.pass_through.safeguards import (
from litellm.llms.anthropic.messages.convertible_blocks import reject_malformed_tool_result_text
from litellm.llms.anthropic.messages.safeguards import (
SafeguardsEvaluator,
build_safeguards_evaluator,
read_classifier_model_setting,
requested_dangerous_tool_use,
with_safeguard_results,
)
from litellm.llms.anthropic.pass_through.utils import (
from litellm.llms.anthropic.messages.utils import (
ProxySubCallLoggingKwargs,
is_reasoning_auto_summary_enabled,
litellm_logging_obj_from_kwargs,
@ -158,7 +158,7 @@ async def prepare_context_managed_request(
drop_invalid_edits: bool = False,
) -> PolyfillResult | None:
"""Apply client compaction history, then optional context_management polyfill."""
from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
from litellm.llms.anthropic.messages.context_management.editors.compact import (
apply_client_compaction_block_history,
)
@ -239,7 +239,7 @@ def _polyfill_will_run(
if edits is None:
return False
from litellm.llms.anthropic.pass_through.context_management.constants import (
from litellm.llms.anthropic.messages.context_management.constants import (
COMPACT_EDIT_TYPE,
)
@ -265,7 +265,7 @@ def _spec_has_non_compact_edits(
if edits is None:
return False
from litellm.llms.anthropic.pass_through.context_management.constants import (
from litellm.llms.anthropic.messages.context_management.constants import (
COMPACT_EDIT_TYPE,
)
@ -298,7 +298,7 @@ def _normalize_spec_edits(
if context_management_explicitly_dropped(additional_drop_params):
return None
from litellm.llms.anthropic.pass_through.context_management.dispatcher import (
from litellm.llms.anthropic.messages.context_management.dispatcher import (
normalize_spec,
)
@ -543,7 +543,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
Handles both string ("max") and dict ({"effort": "max", "summary": ...})
formats. Uses model registry to check supports_xhigh/supports_minimal.
"""
from litellm.llms.anthropic.pass_through.utils import (
from litellm.llms.anthropic.messages.utils import (
normalize_reasoning_effort_value,
)

14
litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py → litellm/llms/anthropic/messages/adapters/streaming_iterator.py
View file

@ -27,8 +27,8 @@ from litellm._uuid import uuid
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
AppliedEdit,
@ -46,7 +46,7 @@ from litellm.types.utils import AdapterCompletionStreamWrapper, ChatCompletionDe
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponseStream
@ -234,7 +234,7 @@ class _CombinedChunkSplitter:
@staticmethod
def _is_combined(chunk: "ModelResponseStream") -> bool:
"""True if ``chunk`` carries response content AND a finish_reason."""
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
@ -1308,7 +1308,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
delta: Final = processed_chunk["delta"]
if delta.get("stop_reason") == "max_tokens":
return processed_chunk
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
refusal_stop_details,
)
@ -1363,7 +1363,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
@staticmethod
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
@ -1400,7 +1400,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
- Different content types in the response
- Specific markers in the content
"""
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)

12
litellm/llms/anthropic/pass_through/adapters/transformation.py → litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -10,8 +10,8 @@ from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part
from litellm.llms.anthropic.pass_through.utils import (
from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part
from litellm.llms.anthropic.messages.utils import (
is_reasoning_auto_summary_enabled,
json_copy_or_original,
prompt_cache_key_from_user_id,
@ -139,14 +139,14 @@ from litellm.llms.anthropic.common_utils import (
normalize_anthropic_tool_use_id,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
from litellm.llms.anthropic.pass_through.context_management import (
from litellm.llms.anthropic.messages.context_management import (
PolyfillResult,
)
from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import (
from litellm.llms.anthropic.messages.native.mid_conversation_system import (
convert_mid_conversation_system_turns,
is_system_role_message,
)
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
refusal_stop_details,
)
@ -210,7 +210,7 @@ from .streaming_iterator import AnthropicSSEStream, AnthropicStreamWrapper
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
from litellm.types.llms.anthropic import ContentBlockContentBlockDict
ToolResultContent: TypeAlias = str | list[ToolMessageContentPart]

0
litellm/llms/anthropic/pass_through/architecture.md → litellm/llms/anthropic/messages/architecture.md
View file

0
litellm/llms/anthropic/pass_through/caller_model_checks.py → litellm/llms/anthropic/messages/caller_model_checks.py
View file

0
litellm/llms/anthropic/pass_through/context_management/__init__.py → litellm/llms/anthropic/messages/context_management/__init__.py
View file

0
litellm/llms/anthropic/pass_through/context_management/constants.py → litellm/llms/anthropic/messages/context_management/constants.py
View file

2
litellm/llms/anthropic/pass_through/context_management/dispatcher.py → litellm/llms/anthropic/messages/context_management/dispatcher.py
View file

@ -18,7 +18,7 @@ from .editors import (
from .result import PolyfillResult
if TYPE_CHECKING:
from litellm.llms.anthropic.pass_through.utils import ProxySubCallLoggingKwargs
from litellm.llms.anthropic.messages.utils import ProxySubCallLoggingKwargs
from litellm.router import Router
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth

0
litellm/llms/anthropic/pass_through/context_management/editors/__init__.py → litellm/llms/anthropic/messages/context_management/editors/__init__.py
View file

0
litellm/llms/anthropic/pass_through/context_management/editors/clear_tool_uses.py → litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py
View file

8
litellm/llms/anthropic/pass_through/context_management/editors/compact.py → litellm/llms/anthropic/messages/context_management/editors/compact.py
View file

@ -21,12 +21,12 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict, Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.llms.anthropic.pass_through.caller_model_checks import (
from litellm.llms.anthropic.messages.caller_model_checks import (
caller_can_call_model,
caller_within_model_budget,
caller_within_model_rate_limit,
)
from litellm.llms.anthropic.pass_through.utils import (
from litellm.llms.anthropic.messages.utils import (
ProxySubCallLoggingKwargs,
proxy_general_settings,
proxy_sub_call_metadata,
@ -269,7 +269,7 @@ def _count_effective_tokens(
threshold check matches the downstream ``input_tokens`` metric.
"""
# Local import to avoid pulling the adapter at module load time.
from litellm.llms.anthropic.pass_through.adapters.transformation import (
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
@ -426,7 +426,7 @@ def _build_summary_messages(
system prompt); the conversation history is translated to OpenAI shape;
the summarization prompt is appended as a final user turn.
"""
from litellm.llms.anthropic.pass_through.adapters.transformation import (
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)

0
litellm/llms/anthropic/pass_through/context_management/errors.py → litellm/llms/anthropic/messages/context_management/errors.py
View file

0
litellm/llms/anthropic/pass_through/context_management/placeholders.py → litellm/llms/anthropic/messages/context_management/placeholders.py
View file

0
litellm/llms/anthropic/pass_through/context_management/result.py → litellm/llms/anthropic/messages/context_management/result.py
View file

2
litellm/llms/anthropic/pass_through/convertible_blocks.py → litellm/llms/anthropic/messages/convertible_blocks.py
View file

@ -12,7 +12,7 @@ from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id
from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
from litellm.types.utils import TokenCountMessagesFormat
_CONVERTIBLE_BLOCK_TYPES: Final = MappingProxyType(

4
litellm/llms/anthropic/pass_through/messages/agentic_streaming_iterator.py → litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py
View file

@ -336,7 +336,7 @@ class AgenticAnthropicStreamingIterator:
await task
async def aclose(self) -> None:
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
aclose_if_supported,
)
@ -379,7 +379,7 @@ class AgenticAnthropicStreamingIterator:
if hasattr(result, "__aiter__"):
self._follow_up_iterator = result.__aiter__()
elif isinstance(result, dict):
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (

2
litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py → litellm/llms/anthropic/messages/native/fake_stream_iterator.py
View file

@ -12,7 +12,7 @@ import json
from collections.abc import Mapping, Sequence
from typing import Final, cast
from litellm.llms.anthropic.pass_through.messages.utils import anthropic_message_delta
from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)

9
litellm/llms/anthropic/pass_through/messages/handler.py → litellm/llms/anthropic/messages/native/handler.py
View file

@ -99,6 +99,7 @@ def _should_route_to_responses_api(
requested_model: str | None = None,
resolved_model: str | None = None,
deployment_model_info: object = None,
stop_sequences: Sequence[str] | None = None,
) -> bool:
"""Return True when the request should use the Responses API path.
@ -107,6 +108,8 @@ def _should_route_to_responses_api(
"""
if litellm.use_chat_completions_url_for_anthropic_messages:
return False
if stop_sequences:
return False
if custom_llm_provider in _RESPONSES_API_PROVIDERS:
return True
if custom_llm_provider is None or requested_model is None or resolved_model is None:
@ -362,7 +365,7 @@ async def _try_websearch_short_circuit(
),
)
if stream:
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
@ -656,7 +659,7 @@ def anthropic_messages_handler(
# reference the provider cannot resolve. Popped from kwargs so it never reaches the provider.
skip_mcp_handler: Final = kwargs.pop("_skip_mcp_handler", False)
if not skip_mcp_handler and tools and may_name_litellm_mcp_gateway(tools): # pyright: ignore[reportUnknownArgumentType] # untyped tools list
from litellm.llms.anthropic.pass_through.messages.mcp_handler import (
from litellm.llms.anthropic.messages.native.mcp_handler import (
anthropic_messages_with_mcp,
)
@ -690,7 +693,7 @@ def anthropic_messages_handler(
if anthropic_messages_provider_config is None:
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
if kwargs.get("compaction") is None and _should_route_to_responses_api(
custom_llm_provider, original_model, model, kwargs.get("model_info")
custom_llm_provider, original_model, model, kwargs.get("model_info"), stop_sequences=stop_sequences
):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=max_tokens,

0
litellm/llms/anthropic/pass_through/messages/interceptors/README.md → litellm/llms/anthropic/messages/native/interceptors/README.md
View file

0
litellm/llms/anthropic/pass_through/messages/interceptors/__init__.py → litellm/llms/anthropic/messages/native/interceptors/__init__.py
View file

2
litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py → litellm/llms/anthropic/messages/native/interceptors/advisor.py
View file

@ -68,7 +68,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
custom_llm_provider: str | None,
**kwargs,
) -> AnthropicMessagesResponse | AsyncIterator:
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)

0
litellm/llms/anthropic/pass_through/messages/interceptors/base.py → litellm/llms/anthropic/messages/native/interceptors/base.py
View file

8
litellm/llms/anthropic/pass_through/messages/mcp_handler.py → litellm/llms/anthropic/messages/native/mcp_handler.py
View file

@ -76,9 +76,9 @@ async def anthropic_messages_with_mcp(
upstream provider as a url it cannot reach.
"""
import litellm
from litellm.llms.anthropic.pass_through.adapters.handler import request_safeguards_evaluator
from litellm.llms.anthropic.pass_through.messages.handler import proxy_classifies_tool_uses
from litellm.llms.anthropic.pass_through.safeguards import with_safeguard_results
from litellm.llms.anthropic.messages.adapters.handler import request_safeguards_evaluator
from litellm.llms.anthropic.messages.native.handler import proxy_classifies_tool_uses
from litellm.llms.anthropic.messages.safeguards import with_safeguard_results
from litellm.mcp_client.tools import (
transform_mcp_tool_to_anthropic_tool,
)
@ -198,7 +198,7 @@ async def anthropic_messages_with_mcp(
else response
)
if stream:
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)

0
litellm/llms/anthropic/pass_through/messages/mid_conversation_system.py → litellm/llms/anthropic/messages/native/mid_conversation_system.py
View file

2
litellm/llms/anthropic/pass_through/messages/response_cache.py → litellm/llms/anthropic/messages/native/response_cache.py
View file

@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Final, cast
import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
AnthropicMessagesStreamingResponse,
BaseAnthropicMessagesStreamingIterator,
aclose_if_supported,

8
litellm/llms/anthropic/pass_through/messages/streaming_iterator.py → litellm/llms/anthropic/messages/native/streaming_iterator.py
View file

@ -18,13 +18,13 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
INCOMPLETE_STREAM_ERROR_MESSAGE,
MESSAGE_DELTA_OPTIONAL_KEYS,
anthropic_message_delta,
)
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
from litellm.proxy.pass_through_endpoints.success_handler import (
PassThroughEndpointLogging,
)
@ -33,7 +33,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointT
from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
if TYPE_CHECKING:
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ: Final = PassThroughEndpointLogging()

6
litellm/llms/anthropic/pass_through/messages/transformation.py → litellm/llms/anthropic/messages/native/transformation.py
View file

@ -10,8 +10,8 @@ from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts
from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards
from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts
from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -736,7 +736,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
litellm_logging_obj: LiteLLMLoggingObj,
) -> AsyncIterator[bytes]:
"""Helper function to handle Anthropic streaming responses using the existing logging handlers"""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
BaseAnthropicMessagesStreamingIterator,
)

2
litellm/llms/anthropic/pass_through/messages/utils.py → litellm/llms/anthropic/messages/native/utils.py
View file

@ -11,7 +11,7 @@ from litellm.llms.anthropic.common_utils import (
sanitize_tool_use_ids_in_anthropic_messages,
strip_empty_content_blocks_from_anthropic_messages,
)
from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled
from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled
from litellm.types.llms.anthropic import (
AnthropicMessagesRequestOptionalParams,
AnthropicStopDetails,

0
litellm/llms/anthropic/pass_through/responses_adapters/__init__.py → litellm/llms/anthropic/messages/responses_adapters/__init__.py
View file

2
litellm/llms/anthropic/pass_through/responses_adapters/handler.py → litellm/llms/anthropic/messages/responses_adapters/handler.py
View file

@ -201,7 +201,7 @@ def _build_responses_kwargs(
if isinstance(reasoning, dict):
effort: Final[object] = reasoning.get("effort")
if isinstance(effort, str):
from litellm.llms.anthropic.pass_through.utils import (
from litellm.llms.anthropic.messages.utils import (
normalize_reasoning_effort_value,
)

8
litellm/llms/anthropic/pass_through/responses_adapters/streaming_iterator.py → litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py
View file

@ -14,13 +14,13 @@ from litellm.exceptions import MidStreamFallbackError
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
INCOMPLETE_STREAM_ERROR_MESSAGE,
refusal_stop_details,
responses_output_refusal_text,
)
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
from litellm.responses.streaming_iterator import (
partial_responses_incomplete_event_on_disconnect,
stream_error_status_and_message,
@ -35,7 +35,7 @@ from .transformation import (
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
from litellm.llms.anthropic.messages.safeguards import SafeguardsEvaluator
from litellm.types.llms.openai import ResponseIncompleteEvent

8
litellm/llms/anthropic/pass_through/responses_adapters/transformation.py → litellm/llms/anthropic/messages/responses_adapters/transformation.py
View file

@ -20,12 +20,12 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.reasoning_effort_utils import (
reasoning_effort_from_thinking,
)
from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_parts
from litellm.llms.anthropic.messages.native.utils import (
refusal_stop_details,
responses_output_refusal_text,
)
from litellm.llms.anthropic.pass_through.utils import (
from litellm.llms.anthropic.messages.utils import (
is_reasoning_auto_summary_enabled,
json_copy_or_original,
prompt_cache_key_from_user_id,
@ -71,7 +71,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if raw_usage is None:
return AnthropicUsage(input_tokens=0, output_tokens=0)
from litellm.llms.anthropic.pass_through.adapters.transformation import (
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
from litellm.responses.utils import ResponseAPILoggingUtils

4
litellm/llms/anthropic/pass_through/safeguards.py → litellm/llms/anthropic/messages/safeguards.py
View file

@ -11,8 +11,8 @@ from typing_extensions import NotRequired, ReadOnly, Unpack, assert_never
import litellm
from litellm._logging import verbose_logger
from litellm.llms.anthropic.pass_through.caller_model_checks import CallerModelVerdict, caller_model_verdict
from litellm.llms.anthropic.pass_through.utils import (
from litellm.llms.anthropic.messages.caller_model_checks import CallerModelVerdict, caller_model_verdict
from litellm.llms.anthropic.messages.utils import (
ProxySubCallLoggingKwargs,
proxy_general_settings,
proxy_sub_call_metadata,

2
litellm/llms/anthropic/pass_through/stream_assembly.py → litellm/llms/anthropic/messages/stream_assembly.py
View file

@ -12,7 +12,7 @@ from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.pass_through.messages.utils import MESSAGE_DELTA_OPTIONAL_KEYS
from litellm.llms.anthropic.messages.native.utils import MESSAGE_DELTA_OPTIONAL_KEYS
from litellm.types.utils import (
Choices,
Message,

0
litellm/llms/anthropic/pass_through/utils.py → litellm/llms/anthropic/messages/utils.py
View file

7
litellm/llms/anthropic/pass_through/__init__.py Normal file
View file

@ -0,0 +1,7 @@
from litellm.litellm_core_utils.renamed_package_shim import alias_renamed_package
__getattr__ = alias_renamed_package(
__name__,
"litellm.llms.anthropic.messages",
nested_renames=((f"{__name__}.messages", "litellm.llms.anthropic.messages.native"),),
)

4
litellm/llms/anthropic/prompt_cache_prediction.py
View file

@ -24,11 +24,11 @@ from litellm.llms.anthropic.common_utils import (
)
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
AnthropicMessagesConfig,
)
from litellm.llms.anthropic.pass_through.messages.utils import AnthropicMessagesRequestUtils, prepare_native_messages
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages
from litellm.router_utils.baseline_request import (
BASELINE_PARAMETERS,
capture_baseline_parameters,

77
litellm/llms/azure/decisions/transformation.py Normal file
View file

@ -0,0 +1,77 @@
import asyncio
from collections.abc import Mapping
from typing import Final
from pydantic import ValidationError
import litellm
from litellm.llms.azure.common_utils import (
BaseAzureLLM,
cached_azure_ad_token_refresh_provider,
)
from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
class AzureDecisionsConfig(OpenAIDecisionsConfig):
"""Azure OpenAI Decisions, with the Azure deployment name in the request body."""
api_key_required = False
def resolve_api_base(self, api_base: str | None) -> str | None:
return BaseAzureLLM.resolve_azure_api_base(api_base)
def resolve_api_key(self, api_key: str | None) -> str | None:
return api_key
def resolve_credential(
self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
) -> str | None:
return api_key
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
try:
params: Final = GenericLiteLLMParams.model_validate({**litellm_params, "api_key": api_key})
except ValidationError as error:
raise litellm.BadRequestError(
message="Invalid Azure Decisions credential parameters",
model=model,
llm_provider="azure",
) from error
clean_headers: Final = super().validate_environment(
{name: value for name, value in headers.items() if name.lower() != "api-key"},
model,
None,
litellm_params,
)
azure_headers: Final = BaseAzureLLM.base_validate_azure_environment(
headers=dict(clean_headers), litellm_params=params
)
if "api-key" in azure_headers:
return azure_headers
if "Authorization" in azure_headers:
return azure_headers
scope: Final = (
params.azure_scope or get_secret_str("AZURE_SCOPE") or "https://cognitiveservices.azure.com/.default"
)
azure_headers["Authorization"] = f"Bearer {cached_azure_ad_token_refresh_provider(scope)()}"
return azure_headers
async def avalidate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None,
litellm_params: Mapping[str, object],
) -> Mapping[str, str]:
return await asyncio.to_thread(self.validate_environment, headers, model, api_key, litellm_params)
def get_complete_url(self, api_base: str, model: str) -> str:
return BaseAzureLLM.get_base_azure_url(api_base, None, route="/openai/v1/decisions")

2
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -5,7 +5,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig
from collections.abc import Sequence
from typing import Any, Final
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.azure.common_utils import BaseAzureLLM

3
litellm/llms/base_llm/decisions/transformation.py
View file

@ -351,9 +351,10 @@ def _ir_score_answer(question: DecisionsIRScoreQuestion, answer: ScoreAnswer) ->
DecisionsIRScoreProbability(
value=index,
label=answer.legend.get(str(index), level.label),
probability=answer.probabilities.get(str(index), 0.0),
probability=answer.probabilities[str(index)],
)
for index, level in enumerate(question.levels)
if str(index) in answer.probabilities
),
extra=MappingProxyType(answer.model_extra or {}),
)

2
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1955,7 +1955,7 @@ class AmazonConverseConfig(BaseConfig):
anthropic_beta_list: list,
) -> None:
"""Keep only compact_20260112 edits for Bedrock; add beta header or drop field."""
from litellm.llms.anthropic.pass_through.context_management.constants import (
from litellm.llms.anthropic.messages.context_management.constants import (
COMPACT_EDIT_TYPE,
)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES

2
litellm/llms/bedrock/claude_platform/messages_transformation.py
View file

@ -1,7 +1,7 @@
from typing import Final
import litellm
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
AnthropicMessagesConfig,
)

2
litellm/llms/bedrock/count_tokens/kimi_k3.py
View file

@ -32,7 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
chat_completion_tools_for,
is_flat_responses_tool,
)
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
KIMI_K3_REPO: Final = "moonshotai/Kimi-K3"

4
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py
View file

@ -24,7 +24,7 @@ from litellm.llms.anthropic.common_utils import (
carried_thinking_keys,
validated_object_mapping,
)
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.base_llm.anthropic_messages.transformation import (
@ -878,7 +878,7 @@ class AmazonAnthropicClaudeMessagesConfig(
merge them from ``message_start`` so logging/cost sees a consistent usage
object (fixes negative input costs: LIT-2411).
"""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
BaseAnthropicMessagesStreamingIterator,
)

2
litellm/llms/bedrock/messages/mantle_transformation.py
View file

@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
AnthropicMessagesConfig,
)

8
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -238,7 +238,7 @@ if TYPE_CHECKING:
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
@ -2650,7 +2650,7 @@ class BaseLLMHTTPHandler:
initial_response: AsyncIterator | AnthropicMessagesResponse
if stream:
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
AnthropicMessagesStreamingResponse,
anthropic_messages_stream_hidden_params,
)
@ -2674,7 +2674,7 @@ class BaseLLMHTTPHandler:
hidden_params=stream_hidden_params,
)
from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import (
AgenticAnthropicStreamingIterator,
)
@ -6237,7 +6237,7 @@ class BaseLLMHTTPHandler:
from typing import cast
from litellm._logging import verbose_logger
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (

33
litellm/llms/databricks/agent/transformation.py
View file

@ -5,16 +5,14 @@ Model Serving, unified: POST {workspace}/serving-endpoints/responses with the en
Databricks Apps: POST {app}/responses
"""
import hashlib
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Final, Protocol
from typing import TYPE_CHECKING, Final
from urllib.parse import quote, urlsplit
import httpx
from pydantic import BaseModel, ConfigDict, StrictStr, TypeAdapter, ValidationError
import litellm
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
@ -41,20 +39,6 @@ _ENDPOINT_URL_SUFFIXES: Final = (_INVOCATIONS_SUFFIX, _APP_RESPONSES_SUFFIX)
_SERVING_ENDPOINTS_SEGMENT: Final = "/serving-endpoints/"
_UNIFIED_RESPONSES_SUFFIX: Final = "/serving-endpoints/responses"
_APP_HOST_SUFFIX: Final = ".databricksapps.com"
_TOKEN_EXPIRY_BUFFER_SECONDS: Final = 60
class _TokenCache(Protocol):
def get_cache(self, key: str) -> object: ...
def set_cache(self, key: str, value: str, *, ttl: int) -> None: ...
def _new_token_cache() -> _TokenCache:
return InMemoryCache(max_size_in_memory=64)
_ENV_M2M_TOKENS: Final = _new_token_cache()
_INPUT_ROLES: Final = frozenset({"system", "developer", "user", "assistant"})
_BODY_PARAMS: Final = (
"max_output_tokens",
@ -220,20 +204,7 @@ def _env_oauth_m2m_token(api_base: str | None) -> str | None:
"set DATABRICKS_HOST to the workspace URL (a Databricks App URL is not a workspace)"
),
)
return _cached_m2m_token(workspace, client_id, client_secret)
def _cached_m2m_token(workspace: str, client_id: str, client_secret: str) -> str:
secret_digest: Final = hashlib.sha256(client_secret.encode()).hexdigest()[:16]
cache_key: Final = f"{workspace}|{client_id}|{secret_digest}"
cached: Final[object] = _ENV_M2M_TOKENS.get_cache(cache_key)
if isinstance(cached, str):
return cached
token: Final = DatabricksBase().get_oauth_m2m_token_and_lifetime(workspace, client_id, client_secret)
ttl: Final = token.expires_in - _TOKEN_EXPIRY_BUFFER_SECONDS
if ttl > 0:
_ENV_M2M_TOKENS.set_cache(cache_key, token.access_token, ttl=ttl)
return token.access_token
return DatabricksBase().get_cached_oauth_m2m_token(workspace, client_id, client_secret)
def _resolved_env_reference(value: str | None) -> str | None:

34
litellm/llms/databricks/common_utils.py
View file

@ -10,21 +10,33 @@ Authentication priority:
3. Databricks SDK automatic auth - Fallback (uses unified auth)
"""
import hashlib
import os
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final, Literal
from typing import Final, Literal, Protocol
from urllib.parse import urlsplit, urlunsplit
from pydantic import BaseModel, ConfigDict
import litellm
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
_DEFAULT_M2M_TOKEN_LIFETIME_SECONDS: Final = 3600
_TOKEN_EXPIRY_BUFFER_SECONDS: Final = 60
class _TokenCache(Protocol):
def get_cache(self, key: str) -> object: ...
def set_cache(self, key: str, value: str, *, ttl: int) -> None: ...
_OAUTH_TOKENS: Final[_TokenCache] = InMemoryCache(max_size_in_memory=64)
class DatabricksException(BaseLLMException):
@ -293,6 +305,26 @@ class DatabricksBase:
) -> str:
return self.get_oauth_m2m_token_and_lifetime(api_base, client_id, client_secret).access_token
def get_cached_oauth_m2m_token(
self,
api_base: str,
client_id: str,
client_secret: str,
*,
token_cache: _TokenCache = _OAUTH_TOKENS,
) -> str:
workspace: Final = self.workspace_root(api_base)
secret_digest: Final = hashlib.sha256(client_secret.encode()).hexdigest()
cache_key: Final = f"{workspace}|{client_id}|{secret_digest}"
cached: Final = token_cache.get_cache(cache_key)
if isinstance(cached, str):
return cached
token: Final = self.get_oauth_m2m_token_and_lifetime(workspace, client_id, client_secret)
ttl: Final = token.expires_in - _TOKEN_EXPIRY_BUFFER_SECONDS
if ttl > 0:
token_cache.set_cache(cache_key, token.access_token, ttl=ttl)
return token.access_token
def get_oauth_m2m_token_and_lifetime(
self,
api_base: str,

13
litellm/llms/databricks/decisions/transformation.py
View file

@ -8,6 +8,8 @@ from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
from litellm.llms.databricks.common_utils import DatabricksBase
from litellm.secret_managers.main import get_secret_str
from litellm.types.decisions import DecisionsIRRequest, UnsupportedDecisionsRequest
DATABRICKS_AI_DECIDE_MODEL: Final = "ai_decide"
@ -74,6 +76,17 @@ class DatabricksDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("DATABRICKS_API_KEY", "DATABRICKS_TOKEN")
api_base_env = ("DATABRICKS_API_BASE",)
def resolve_credential(
self, api_key: str | None, api_base: str, litellm_params: Mapping[str, object]
) -> str | None:
if api_key:
return api_key
client_id: Final = get_secret_str("DATABRICKS_CLIENT_ID")
client_secret: Final = get_secret_str("DATABRICKS_CLIENT_SECRET")
if not client_id or not client_secret:
return None
return DatabricksBase().get_cached_oauth_m2m_token(api_base, client_id, client_secret)
def missing_api_base_message(self, custom_llm_provider: str) -> str:
return (
f"api_base is required for Decisions provider '{custom_llm_provider}': set DATABRICKS_API_BASE to "

2
litellm/llms/databricks/messages/transformation.py
View file

@ -8,7 +8,7 @@ from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.databricks.common_utils import (

2
litellm/llms/deepseek/messages/transformation.py
View file

@ -9,7 +9,7 @@ from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str

2
litellm/llms/github_copilot/messages/transformation.py
View file

@ -3,7 +3,7 @@ from typing import Final, cast # noqa: TID251 # narrows untyped request dicts
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)

16
litellm/llms/lm_studio/decisions/transformation.py Normal file
View file

@ -0,0 +1,16 @@
from typing import Final
from litellm.constants import PLACEHOLDER_API_KEY
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class LMStudioDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("LM_STUDIO_API_KEY",)
api_base_env = ("LM_STUDIO_API_BASE",)
api_key_required = False
supports_images = True
inlines_remote_images = True
def resolve_api_key(self, api_key: str | None) -> str | None:
resolved: Final = super().resolve_api_key(api_key)
return None if resolved == PLACEHOLDER_API_KEY else resolved

2
litellm/llms/minimax/messages/transformation.py
View file

@ -4,7 +4,7 @@ MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's
import litellm
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str

10
litellm/llms/openai/decisions/transformation.py
View file

@ -270,10 +270,9 @@ def _openai_choice_answer(question: DecisionsIRChoiceQuestion, answer: Decisions
name=question.name,
choice=answer.choice,
probabilities=tuple(
OpenAIChoiceProbability(
value=option.value, probability=probabilities.get(systemone_choice_key(option.value), 0.0)
)
OpenAIChoiceProbability(value=option.value, probability=probabilities[systemone_choice_key(option.value)])
for option in question.choices
if systemone_choice_key(option.value) in probabilities
),
confidence=answer.confidence,
)
@ -285,10 +284,9 @@ def _openai_score_answer(question: DecisionsIRScoreQuestion, answer: DecisionsIR
name=question.name,
score=answer.score,
probabilities=tuple(
OpenAIScoreProbability(
value=index, label=decisions_text(level.label), probability=probabilities.get(index, 0.0)
)
OpenAIScoreProbability(value=index, label=decisions_text(level.label), probability=probabilities[index])
for index, level in enumerate(question.levels)
if index in probabilities
),
confidence=answer.confidence,
)

4
litellm/llms/openai/responses/count_tokens/token_counter.py
View file

@ -15,8 +15,8 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
from litellm.llms.base_llm.base_utils import BaseTokenCounter

6
litellm/llms/openai/responses/count_tokens/transformation.py
View file

@ -15,9 +15,9 @@ from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error
from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
from litellm.llms.anthropic.messages.convertible_blocks import malformed_tool_result_text_error
from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
from litellm.responses.utils import ResponsesAPIRequestUtils

2
litellm/llms/openai_like/messages/transformation.py
View file

@ -3,7 +3,7 @@ from typing import Final
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.openai_like.json_loader import SimpleProviderConfig

2
litellm/llms/opencode/messages/transformation.py
View file

@ -3,7 +3,7 @@ from collections.abc import Mapping, Sequence
from pydantic import TypeAdapter
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig
from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig
from litellm.llms.opencode.common_utils import go_api_base, go_headers

2
litellm/llms/tencent/messages/transformation.py
View file

@ -7,7 +7,7 @@ alongside its standard OpenAI-compatible chat completions endpoint.
import litellm
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.secret_managers.main import get_secret_str

14
litellm/llms/vertex_ai/count_tokens/context.py
View file

@ -134,7 +134,7 @@ def vertex_gemini_count_context(
def unconvertible_messages_error(
messages: Sequence[Mapping[str, object]] | None, messages_format: TokenCountMessagesFormat
) -> VertexGeminiCountContextError | None:
from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error
from litellm.llms.anthropic.messages.convertible_blocks import unconvertible_blocks_error
error_message: Final = unconvertible_blocks_error(messages, messages_format)
return None if error_message is None else VertexGeminiCountContextError(error_message=error_message)
@ -145,7 +145,7 @@ def malformed_tool_result_text_count_error(
messages_format: TokenCountMessagesFormat,
deployment: Mapping[str, object] | None,
) -> VertexGeminiCountContextError | None:
from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_count_error
from litellm.llms.anthropic.messages.convertible_blocks import malformed_tool_result_count_error
error_message: Final = malformed_tool_result_count_error(messages, messages_format, deployment)
return None if error_message is None else VertexGeminiCountContextError(error_message=error_message)
@ -235,7 +235,7 @@ def _view_without_edits(
system: object | None,
typed_system: str | Sequence[Mapping[str, object]] | None,
) -> InferenceRequestView:
from litellm.llms.anthropic.pass_through.context_management.editors.compact import (
from litellm.llms.anthropic.messages.context_management.editors.compact import (
apply_client_compaction_block_history,
)
@ -253,9 +253,9 @@ def _view_without_edits(
async def _anthropic_inference_view(
messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits
) -> InferenceRequestView | VertexGeminiCountContextError:
from litellm.llms.anthropic.pass_through.adapters.handler import prepare_context_managed_request
from litellm.llms.anthropic.pass_through.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
from litellm.llms.anthropic.messages.adapters.handler import prepare_context_managed_request
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
typed_system: Final = _typed_system(system)
if isinstance(typed_system, VertexGeminiCountContextError):
@ -308,7 +308,7 @@ def _chat_messages(
model: str,
custom_llm_provider: Literal["vertex_ai", "gemini"],
) -> Sequence[AllMessageValues]:
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
match messages_format:
case "anthropic":

3
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from litellm.litellm_core_utils.renamed_package_shim import alias_renamed_package
__getattr__ = alias_renamed_package(__name__, "litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages")

94
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py → litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py
View file

@ -1,8 +1,8 @@
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from typing import Final
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.pass_through.messages.transformation import (
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
messages_carry_output_config,
)
@ -26,34 +26,92 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
def should_strip_billing_metadata(self) -> bool:
return True
@staticmethod
def _vertex_auth_parameters(
litellm_params: dict[str, object], # mutable-ok: delegates to the existing Vertex parameter readers
) -> tuple[str | None, str | None, str | None]:
return (
VertexBase.safe_get_vertex_ai_project(litellm_params),
VertexBase.safe_get_vertex_ai_location(litellm_params),
VertexBase.safe_get_vertex_ai_credentials(litellm_params),
)
def validate_anthropic_messages_environment(
self,
headers: dict,
headers: dict[str, str],
model: str,
messages: Sequence[object],
optional_params: dict,
litellm_params: dict,
optional_params: dict[str, object],
litellm_params: dict[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> tuple[dict, str | None]:
) -> tuple[dict[str, str], str | None]:
"""
OPTIONAL
Validate the environment for the request
"""
# Work on a local copy — router shallow-copies litellm_params so the caller's
# headers dict may be the shared deployment extra_headers object.
headers = dict(headers)
vertex_ai_project: Final = VertexBase.safe_get_vertex_ai_project(litellm_params)
vertex_ai_location: Final = VertexBase.safe_get_vertex_ai_location(litellm_params)
vertex_credentials: Final = VertexBase.safe_get_vertex_ai_credentials(litellm_params)
vertex_ai_project, vertex_ai_location, vertex_credentials = self._vertex_auth_parameters(litellm_params)
access_token, project_id = self._ensure_access_token(
credentials=vertex_credentials,
project_id=vertex_ai_project,
custom_llm_provider="vertex_ai",
)
headers["Authorization"] = f"Bearer {access_token}"
return self._finalize_vertex_messages_environment(
headers,
model,
messages,
optional_params,
api_base,
access_token,
project_id,
vertex_ai_project,
vertex_ai_location,
)
async def avalidate_anthropic_messages_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors the provider validation contract
model: str,
messages: Sequence[object],
optional_params: dict[str, object], # mutable-ok: mirrors the provider validation contract
litellm_params: dict[str, object], # mutable-ok: mirrors the provider validation contract
api_key: str | None = None,
api_base: str | None = None,
) -> tuple[dict[str, str], str | None]: # mutable-ok: the caller consumes request headers
vertex_ai_project, vertex_ai_location, vertex_credentials = self._vertex_auth_parameters(litellm_params)
access_token, project_id = await self._ensure_access_token_async(
credentials=vertex_credentials,
project_id=vertex_ai_project,
custom_llm_provider="vertex_ai",
)
return self._finalize_vertex_messages_environment(
headers,
model,
messages,
optional_params,
api_base,
access_token,
project_id,
vertex_ai_project,
vertex_ai_location,
)
def _finalize_vertex_messages_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[object],
optional_params: dict, # mutable-ok: preserves the existing provider optional-parameter contract
api_base: str | None,
access_token: str,
project_id: str,
vertex_ai_project: str | None,
vertex_ai_location: str | None,
) -> tuple[dict[str, str], str | None]: # mutable-ok: the caller consumes request headers
# Router shallow-copies litellm_params, so headers may be shared between requests.
request_headers: Final = dict(headers)
request_headers["Authorization"] = f"Bearer {access_token}"
api_base = self.get_complete_vertex_url(
custom_api_base=api_base,
@ -65,14 +123,14 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
model=model,
)
headers["content-type"] = "application/json"
request_headers["content-type"] = "application/json"
# Add beta headers for Vertex AI
tools: Final = optional_params.get("tools", [])
beta_values: Final[set[str]] = set()
# Get existing beta headers if any
existing_beta: Final = headers.get("anthropic-beta")
existing_beta: Final = request_headers.get("anthropic-beta")
if existing_beta:
beta_values.update(b.strip() for b in existing_beta.split(","))
@ -117,9 +175,9 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value)
if beta_values:
headers["anthropic-beta"] = ",".join(beta_values)
request_headers["anthropic-beta"] = ",".join(beta_values)
return headers, api_base
return request_headers, api_base
def get_complete_url(
self,

2
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py
View file

@ -2,7 +2,7 @@
Shared sanitization for ``output_config`` / ``output_format`` on Vertex AI
Claude. Lives in its own module so both the chat-completion transformation
(``transformation.py``) and the Messages pass-through transformation
(``experimental_pass_through/transformation.py``) can import it without
(``messages/transformation.py``) can import it without
forming a cycle through the parent module's heavier imports.
CodeQL flagged the ``..transformation`` import path as a potential cyclic

2
litellm/messages/dispatch.py
View file

@ -5,7 +5,7 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele
from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.anthropic.pass_through.messages import handler as main
from litellm.llms.anthropic.messages.native import handler as main
from litellm.rust_bridge import detectors
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
from litellm.rust_bridge.dispatch import PublicDispatch

2
litellm/proxy/anthropic_endpoints/endpoints.py
View file

@ -19,7 +19,7 @@ from litellm.anthropic_interface.exceptions import (
)
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.tool_fields import first_validation_error
from litellm.llms.anthropic.pass_through.context_management import (
from litellm.llms.anthropic.messages.context_management import (
AnthropicContextManagementError,
)
from litellm.llms.base_llm.guardrail_translation.utils import (

1
litellm/proxy/common_request_processing.py
View file

@ -4687,6 +4687,7 @@ class ProxyBaseLLMRequestProcessing:
total_tokens=total_tokens,
completion_tokens_details=usage.get("completion_tokens_details"),
prompt_tokens_details=usage.get("prompt_tokens_details"),
server_tool_use=usage.get("server_tool_use"),
)
@staticmethod

16
litellm/proxy/guardrails/anthropic_sse.py
View file

@ -13,15 +13,15 @@ from types import MappingProxyType
from typing import Final
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.pass_through.stream_assembly import anthropic_message_delta_fields
from litellm.llms.anthropic.pass_through.stream_assembly import (
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields
from litellm.llms.anthropic.messages.stream_assembly import (
assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.pass_through.stream_assembly import (
from litellm.llms.anthropic.messages.stream_assembly import (
is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.pass_through.stream_assembly import (
from litellm.llms.anthropic.messages.stream_assembly import (
is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.types.utils import Choices, ModelResponse
@ -50,7 +50,7 @@ def anthropic_sse_error_frames(message: str) -> tuple[bytes, ...]:
)
from litellm.llms.anthropic.pass_through.stream_assembly import ( # noqa: E402 # re-export after the definitions above
from litellm.llms.anthropic.messages.stream_assembly import ( # noqa: E402 # re-export after the definitions above
is_sse_error_stream as is_sse_error_stream, # noqa: PLC0414 # public re-export
)
@ -111,10 +111,10 @@ def anthropic_sse_chunks_from_response(
the input the classifier judged. Their free-text ``explanation`` was written about the text the
guardrail rewrote, so it is dropped unless the caller rewrites no text (``keep_explanations``).
"""
from litellm.llms.anthropic.pass_through.adapters.transformation import (
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)

2
litellm/proxy/guardrails/guardrail_hooks/straiker/straiker.py
View file

@ -559,7 +559,7 @@ def _v3_answer(request_data: Mapping[str, object], model: str | None) -> Mapping
return _v3_text_completion_as_chat(response)
if not isinstance(response, ModelResponse) or not _v3_anthropic_messages_route(request_data):
return _jsonable_dict(response)
from litellm.llms.anthropic.pass_through.adapters.transformation import (
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)

2
litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py
View file

@ -21,7 +21,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_content_from_model_response,
)
from litellm.llms.anthropic import get_anthropic_config
from litellm.llms.anthropic.pass_through.stream_assembly import (
from litellm.llms.anthropic.messages.stream_assembly import (
build_complete_streaming_response,
build_complete_streaming_response_legacy,
build_usage_only_response_from_chunks,

4
litellm/proxy/pass_through_endpoints/streaming_handler.py
View file

@ -311,7 +311,7 @@ class PassThroughStreamingHandler:
@staticmethod
def _error_frame_closed_at(endpoint_type: EndpointType, raw_bytes: Sequence[bytes]) -> tuple[str, str, int] | None:
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
closing_superseded_attempt,
parse_anthropic_error_event,
)
@ -342,7 +342,7 @@ class PassThroughStreamingHandler:
- Vertex AI
- OpenAI
"""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
is_message_stop_chunk, # pyright: ignore[reportPrivateUsage] # both native stream paths share terminal-event detection
is_provider_error_chunk, # pyright: ignore[reportPrivateUsage] # provider errors must not become cache evidence
)

4
litellm/proxy/proxy_server.py
View file

@ -374,7 +374,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import (
SensitiveDataMasker,
mask_sensitive_keys,
)
from litellm.llms.anthropic.pass_through.convertible_blocks import (
from litellm.llms.anthropic.messages.convertible_blocks import (
deployment_drops_params,
drop_non_string_tool_result_texts,
malformed_tool_result_count_error,
@ -15171,7 +15171,7 @@ def _router_default_litellm_params(router: Router | None) -> Mapping[str, object
def _count_context_management(
context_management: JsonValue, model: str, request_litellm_params: Mapping[str, JsonValue]
) -> CountContextManagement | None:
from litellm.llms.anthropic.pass_through.adapters.handler import context_management_explicitly_dropped
from litellm.llms.anthropic.messages.adapters.handler import context_management_explicitly_dropped
if context_management is None or llm_router is None:
return None

30
litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
View file

@ -234,6 +234,9 @@ class DefaultTeamSettingsResponse(SettingsResponse):
class UIThemeSettingsResponse(SettingsResponse):
"""Response model for UI theme settings"""
logo_url_via_proxy: bool = False
logo_url_dark_via_proxy: bool = False
_TEAM_ADMIN_FIELD_ENUM: Final = tuple(sorted(SUPPORTED_TEAM_ADMIN_PERMISSIONS))
@ -1377,7 +1380,7 @@ async def update_sso_settings(
tags=["UI Theme Settings"],
response_model=UIThemeSettingsResponse,
)
async def get_ui_theme_settings():
async def get_ui_theme_settings() -> UIThemeSettingsResponse:
"""
Get UI theme configuration from the litellm_settings.
Returns current logo settings for UI customization.
@ -1390,18 +1393,23 @@ async def get_ui_theme_settings():
# Load existing config
config: Final = await proxy_config.get_config()
result: Final = await _get_settings_with_schema(
settings_key="ui_theme_config",
settings_class=UIThemeConfig,
config=config,
result: Final = _SettingsWithSchema.model_validate(
await _get_settings_with_schema(
settings_key="ui_theme_config",
settings_class=UIThemeConfig,
config=config,
)
)
stored_values: Final = result.get("values", {})
result["values"] = {
**stored_values,
**{field: _resolve_ui_theme_field(stored_values, field) for field in _UI_THEME_FIELD_ENV_VARS},
values: Final = {
**result.values,
**{field: _resolve_ui_theme_field(result.values, field) for field in _UI_THEME_FIELD_ENV_VARS},
}
return result
return UIThemeSettingsResponse(
values=values,
field_schema=result.field_schema,
logo_url_via_proxy=values["logo_url"] is None and bool(os.getenv("UI_LOGO_PATH", "").strip()),
logo_url_dark_via_proxy=values["logo_url_dark"] is None and bool(os.getenv("UI_LOGO_PATH_DARK", "").strip()),
)
def _validate_public_image_url(value: str | None, field_name: str) -> None:

22
litellm/router.py
View file

@ -737,7 +737,7 @@ def _anthropic_stream_forwards_ping_live(chunk: object, has_generated_content: b
"""A `ping` keepalive reaches the client live whenever the stream has not committed: it carries no
lifecycle, so it cannot create overlapping lifecycles on the wire, and it keeps the connection alive
while lifecycle frames sit buffered for a possible fallback during a long thinking pass."""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import is_anthropic_ping_chunk
from litellm.llms.anthropic.messages.native.streaming_iterator import is_anthropic_ping_chunk
return not has_generated_content and is_anthropic_ping_chunk(chunk)
@ -753,7 +753,7 @@ def _without_line_breaks(value: object) -> str:
def _anthropic_stream_error_is_gateway_verdict(chunk: object) -> bool:
"""AgenticAnthropicStreamingIterator's own retrieval-failure frame is the gateway's verdict, not a provider
failure: another deployment would rerun the same failed hook, so it reaches the client instead of falling back."""
from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import (
is_server_fulfilled_tool_leak_error,
)
@ -917,7 +917,7 @@ def _stream_commits_now(chunk: object, has_generated_content: bool, buffered_chu
pre-content buffer cap was hit) rather than keep buffering lifecycle
frames toward a possible fallback.
"""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
is_anthropic_content_delta_chunk,
)
@ -6307,7 +6307,7 @@ class Router:
response=response,
kwargs=kwargs,
):
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
safeguard_refusal_error,
)
@ -6424,16 +6424,16 @@ class Router:
including pings and provider error frames, is forwarded live and
verbatim instead.
"""
from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import (
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
aclose_if_supported,
aclose_superseded_attempt,
parse_anthropic_error_event,
parse_anthropic_refusal_stop_details,
)
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
safeguard_refusal_error,
)
@ -6797,7 +6797,7 @@ class Router:
A retry stamps the retry headers async_function_with_retries would have for a pre-stream retry;
a later hop the recovered stream makes replaces them with its own, the way a fallback's do.
"""
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
aclose_if_supported,
anthropic_messages_response_as_sse_events,
)
@ -9931,7 +9931,7 @@ class Router:
when a content-policy fallback is configured; a plain refusal without stop_details, or
any response with nothing configured, is returned to the client unchanged.
"""
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
get_safeguard_refusal_stop_details,
)
@ -13136,6 +13136,7 @@ class Router:
healthy_deployments=remaining,
request_kwargs=tag_request,
metadata_variable_name=metadata_variable_name,
request_deployments=request_deployments,
)
except ValueError: # tag routing sends this request to none of the rest
break
@ -14147,7 +14148,7 @@ class Router:
`tools` (Chat Completions, Responses and Anthropic Messages shapes) and the
Anthropic Messages top-level `system` block.
"""
from litellm.llms.anthropic.pass_through.messages.utils import (
from litellm.llms.anthropic.messages.native.utils import (
anthropic_system_to_openai_message,
)
@ -15100,6 +15101,7 @@ class Router:
request_kwargs=request_kwargs,
healthy_deployments=healthy_deployments,
metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs),
request_deployments=request_deployments,
)
# narrow to whatever `self.routing_plugins` left in candidate_models

23
litellm/router_strategy/tag_based_routing.py
View file

@ -9,10 +9,12 @@ Use this to route requests between Teams
"""
import re
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload
from typing_extensions import ReadOnly
from litellm._logging import verbose_logger
from litellm.constants import (
@ -56,6 +58,10 @@ class _DeploymentLike(Protocol):
def get(self, key: Literal["model_name"], /) -> object: ...
class _DeploymentWithModelInfo(TypedDict, total=False):
model_info: ReadOnly[Mapping[str, object]]
class _MetadataLike(Protocol):
@overload
def get(self, key: Literal["tags"], /) -> Sequence[str] | None: ...
@ -285,13 +291,14 @@ def _unknown_required_tag_hides_an_answer(
def _chain_allows_fail_open(
healthy_deployments: _DeploymentPool,
request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
excluded_set: frozenset[str],
required_set: frozenset[str],
routing_confirmed: frozenset[str],
) -> bool:
if _unknown_required_tag_hides_an_answer(healthy_deployments, excluded_set, required_set, routing_confirmed):
return False
return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in healthy_deployments)
return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in request_deployments)
def _trusted_only_pool(
@ -327,6 +334,7 @@ def _trusted_only_pool(
def _resolve_or_fail_open(
request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
pool: Sequence[_DeploymentLike],
healthy_deployments: _DeploymentPool,
excluded_set: frozenset[str],
@ -339,7 +347,7 @@ def _resolve_or_fail_open(
) -> tuple[_DeploymentLike, ...]:
if pool:
return tuple(pool)
if _chain_allows_fail_open(healthy_deployments, excluded_set, required_set, routing_confirmed):
if _chain_allows_fail_open(healthy_deployments, request_deployments, excluded_set, required_set, routing_confirmed):
# Fall open only within whatever still satisfies whichever constraints
# trace back to key/team policy. A constraint with no inherited backing at
# all (or, when inherited_tags is unavailable, any constraint at all) can
@ -356,6 +364,7 @@ def _resolve_or_fail_open(
def _resolve_constraint_only_pool(
request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
healthy_deployments: _DeploymentPool,
excluded_set: frozenset[str],
required_set: frozenset[str],
@ -371,6 +380,7 @@ def _resolve_constraint_only_pool(
else _exclude_deployments(_default_tagged_pool(healthy_deployments), excluded_set)
)
return _resolve_or_fail_open(
request_deployments,
pool,
healthy_deployments,
excluded_set,
@ -553,6 +563,8 @@ async def get_deployments_for_tag(
healthy_deployments: _DeploymentPool,
request_kwargs: _RequestKwargsLike | None = None,
metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata",
*,
request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
) -> _DeploymentPool:
"""
Returns a list of deployments that match the requested model and tags in the request.
@ -624,6 +636,7 @@ async def get_deployments_for_tag(
if constraint_only:
return _resolve_constraint_only_pool(
request_deployments,
healthy_deployments,
excluded_set,
required_set,
@ -675,6 +688,7 @@ async def get_deployments_for_tag(
if len(new_healthy_deployments) == 0 and len(default_deployments) == 0:
return _resolve_or_fail_open(
request_deployments,
(),
healthy_deployments,
excluded_set,
@ -692,6 +706,7 @@ async def get_deployments_for_tag(
and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed)
):
return _resolve_or_fail_open(
request_deployments,
(),
healthy_deployments,
excluded_set,

2
litellm/router_utils/baseline_request.py
View file

@ -7,7 +7,7 @@ from typing import Final, cast
from pydantic import JsonValue, TypeAdapter, ValidationError
from litellm.llms.anthropic.pass_through.messages.utils import anthropic_messages_optional_param_keys
from litellm.llms.anthropic.messages.native.utils import anthropic_messages_optional_param_keys
CACHE_SETTINGS: Final = (
"system",

2
litellm/rust_bridge/host/callbacks_legacy_python.py
View file

@ -377,7 +377,7 @@ def stream_success(
end: datetime.datetime,
first_chunk: datetime.datetime | None,
) -> None:
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
)
from litellm.proxy.pass_through_endpoints.streaming_handler import PassThroughStreamingHandler

4
litellm/rust_bridge/host/routes/messages.py
View file

@ -10,7 +10,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
from litellm.llms.anthropic.pass_through.utils import is_reasoning_auto_summary_enabled
from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled
from litellm.rust_bridge import failures
from litellm.rust_bridge.public_call import known_params, optional_str
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
@ -64,7 +64,7 @@ def body_params(request: Mapping[str, object]) -> Mapping[str, object]:
def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]:
from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
from litellm.llms.anthropic.messages.native.streaming_iterator import (
anthropic_messages_stream_hidden_params,
)

4
litellm/rust_bridge/messages/detectors.py
View file

@ -73,7 +73,7 @@ def _resolved_provider(request: NativeCall) -> str | None:
def intercepted(request: NativeCall) -> bool:
"""The async Messages handler delegates matching tools to a registered interceptor."""
from litellm.llms.anthropic.pass_through.messages.interceptors import get_messages_interceptors
from litellm.llms.anthropic.messages.native.interceptors import get_messages_interceptors
tools: Final = optional_sequence(request.resolved.get("tools"))
if not tools:
@ -85,6 +85,6 @@ def intercepted(request: NativeCall) -> bool:
def safeguards_classifier(request: NativeCall) -> bool:
"""Python's proxy classifier evaluates dangerous tool use for non-Claude models."""
from litellm.llms.anthropic.pass_through.safeguards import native_route_classifies
from litellm.llms.anthropic.messages.safeguards import native_route_classifies
return native_route_classifies(str(request.resolved["model"]), request.resolved.get("safeguards"))

12
litellm/utils.py
View file

@ -9610,12 +9610,18 @@ class ProviderConfigManager:
return litellm.DatabricksDecisionsConfig()
if provider == LlmProviders.HOSTED_VLLM:
return litellm.HostedVLLMDecisionsConfig()
if provider == LlmProviders.LM_STUDIO:
return litellm.LMStudioDecisionsConfig()
if provider == LlmProviders.LAYA:
return litellm.LayaDecisionsConfig()
if provider == LlmProviders.BESPOKE:
return litellm.BespokeDecisionsConfig()
if provider == LlmProviders.OPENAI:
return litellm.OpenAIDecisionsConfig()
if provider in (LlmProviders.OPENAI, LlmProviders.AZURE):
configs: Final = {
LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig,
LlmProviders.AZURE: litellm.AzureDecisionsConfig,
}
return configs[provider]()
if provider == LlmProviders.AZURE_AI:
return litellm.AzureAIDecisionsConfig()
return None
@ -9662,7 +9668,7 @@ class ProviderConfigManager:
return BedrockMantleAnthropicMessagesConfig()
elif litellm.LlmProviders.VERTEX_AI == provider:
if "claude" in model_lower:
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import (
VertexAIPartnerModelsAnthropicMessagesConfig,
)

4
ruff-strict.toml
View file

@ -71,7 +71,7 @@ max-args = 5
# directly, each with a `# noqa: TID251 # <reason>`.
"litellm.responses.main.responses".msg = "Import litellm.responses.dispatch.responses so the call routes through dispatch."
"litellm.responses.main.aresponses".msg = "Import litellm.responses.dispatch.aresponses so the call routes through dispatch."
"litellm.llms.anthropic.pass_through.messages.handler.anthropic_messages".msg = "Import litellm.messages.anthropic_messages so the call routes through dispatch."
"litellm.llms.anthropic.pass_through.messages.handler.anthropic_messages_handler".msg = "Import litellm.messages.anthropic_messages_handler so the call routes through dispatch."
"litellm.llms.anthropic.messages.native.handler.anthropic_messages".msg = "Import litellm.messages.anthropic_messages so the call routes through dispatch."
"litellm.llms.anthropic.messages.native.handler.anthropic_messages_handler".msg = "Import litellm.messages.anthropic_messages_handler so the call routes through dispatch."
"litellm.main.completion".msg = "Import litellm.completion so the call routes through dispatch."
"litellm.main.acompletion".msg = "Import litellm.acompletion so the call routes through dispatch."

4
tests/e2e/coverage_registry/llm_conversational.yaml
View file

@ -76,8 +76,8 @@
- {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven}
- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
- {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"}
- {id: llm.responses.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.9 / LIT-4778", rationale: "Responses missing/empty input and missing model are rejected"}
- {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}

6
tests/e2e/coverage_registry/quota_management.yaml
View file

@ -41,7 +41,7 @@
- {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"}
- {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"}
- {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"}
- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"}
- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"}
- {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"}
- {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"}
- {id: quota_management.spend_tracking.discount_config.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: discount_config, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/management_endpoints/cost_tracking_settings.py + cost_calculator.py", rationale: "A persisted provider discount changes the real request's spend row"}
@ -62,11 +62,11 @@
- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"}
- {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"}
- {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"}
- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"}
- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"}
- {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"}
- {id: quota_management.spend_tracking.service_tier_stream.records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", fail_before_fix: proven, rationale: "A streamed call with no service_tier requested bills at the rates of the tier OpenAI stamps on its chunks and records that served tier on the row; the reassembled stream dropped the provider tier so the row recorded none and priced at the default rates"}
- {id: quota_management.spend_tracking.service_tier_stream.responses_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [responses], source: "responses/streaming_iterator.py", rationale: "A streamed /v1/responses call bills at the tier carried on the response.completed event's inner response and records that served tier on the spend row"}
- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/pass_through/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"}
- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/messages/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"}
- {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"}
- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"}
- {id: quota_management.spend_tracking.websearch_interception.bills_under_request_session, module: quota_management, tier: P1, behavior: spend_tracking, variant: websearch_interception, assertions: [bills_under_request_session], exercised_on: [messages], source: "integrations/websearch_interception/handler.py", fail_before_fix: proven, rationale: "A web_search server tool the proxy intercepts into litellm.asearch writes its own asearch spend row, and that row carries the parent request's session_id so the session view counts the search and its cost next to the turn that triggered it (LIT-8063)"}

2
tests/integration/messages_endpoint/test_guardrail_long_tool_names_wire.py
View file

@ -14,7 +14,7 @@ from integration._support.ports import allocate_port
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
from litellm.llms.anthropic.pass_through.adapters.transformation import truncate_tool_name
from litellm.llms.anthropic.messages.adapters.transformation import truncate_tool_name
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_JSON_OBJECTS: Final = TypeAdapter(list[dict[str, JsonValue]])

Some files were not shown because too many files have changed in this diff Show more