scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-02 07:13:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 150

Merge branch 'main' into litellm-providers/price-sync-databricks

Browse source
This commit is contained in:
Devin AI 2026-10-31 13:09:59 +00:00
commit 2fde028759
349 changed files with 18262 additions and 3372 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.circleci/AGENTS.md Normal file
View file

@ -0,0 +1,7 @@
# .circleci
The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values
Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base
A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base

58
.circleci/config.yml
View file

@ -21,19 +21,30 @@ orbs:
win: circleci/windows@5.0 # Add Windows orb
commands:
checkout_migration_source:
checkout_over_https:
parameters:
revision:
type: string
default: ""
shell:
type: string
default: /bin/bash -eo pipefail
steps:
- run:
name: Select the requested migration test revision
name: Check out the public repository over HTTPS
shell: << parameters.shell >>
environment:
MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
REQUESTED_REVISION: << parameters.revision >>
command: |
revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
git init
git remote add origin https://github.com/BerriAI/litellm.git
git fetch --depth 1 origin "$revision"
git checkout --detach FETCH_HEAD
for attempt in 1 2 3; do
git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break
[[ $attempt -lt 3 ]] || exit 1
sleep 5
done
skip_if_unrelated_changes:
parameters:
category:
@ -347,7 +358,7 @@ commands:
- when:
condition: << parameters.checkout >>
steps:
- checkout
- checkout_over_https
- setup_google_dns
- install_uv
- install_rust
@ -377,7 +388,8 @@ jobs:
CARGO_HTTP_MULTIPLEXING: "false"
CARGO_NET_RETRY: "5"
steps:
- checkout
- checkout_over_https:
shell: bash.exe
- run:
name: Skip job when no windows-release-relevant files changed
shell: bash.exe
@ -451,7 +463,7 @@ jobs:
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- setup_google_dns
- install_uv
- install_rust
@ -482,7 +494,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
@ -559,7 +571,7 @@ jobs:
resource_class: medium
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -659,7 +671,7 @@ jobs:
resource_class: large
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -740,7 +752,7 @@ jobs:
resource_class: large
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes:
category: client
- setup_google_dns
@ -794,7 +806,7 @@ jobs:
# the rest of the suite: nothing else performs a logout.
PROXY_LOGOUT_URL: "https://www.example.com"
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -940,7 +952,7 @@ jobs:
# and by migration.serverRootPath.config.ts, which refuses to run without it.
SERVER_ROOT_PATH: "/litellm"
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -1066,11 +1078,12 @@ jobs:
- when:
condition: << parameters.migration_qualification >>
steps:
- checkout_migration_source
- checkout_over_https:
revision: << pipeline.parameters.migration_source_sha >>
- unless:
condition: << parameters.migration_qualification >>
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- run:
@ -1122,7 +1135,8 @@ jobs:
MIGRATION_TEST_OUTPUT: /tmp/migration-results
PYTHONPATH: tests/e2e
steps:
- checkout_migration_source
- checkout_over_https:
revision: << pipeline.parameters.migration_source_sha >>
- run:
name: Generate LiteLLM master key
command: |
@ -1197,7 +1211,7 @@ jobs:
resource_class: medium
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
@ -1371,7 +1385,7 @@ jobs:
environment:
UV_PYTHON_DOWNLOADS: never
steps:
- checkout
- checkout_over_https
- install_uv
- run:
name: Install the locked shard verifier and coverage dependencies
@ -1419,7 +1433,7 @@ jobs:
ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
command: |
[[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE"
git rm -r -f --quiet litellm enterprise litellm-proxy-extras
git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
git reset --quiet
@ -1494,7 +1508,7 @@ jobs:
environment:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- setup_litellm_test_deps:
checkout: false

2
litellm-rust/crates/core-utils/src/core_helpers.rs
View file

@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
("max_tokens", "length"),
("refusal", "content_filter"),
("compaction", "length"),
("pause_turn", "length"),
("model_context_window_exceeded", "length"),
("guardrail_intervened", "content_filter"),
("content_filtered", "content_filter"),
@ -75,6 +76,7 @@ mod tests {
#[case::end_turn("end_turn", "stop")]
#[case::stop_sequence("stop_sequence", "stop")]
#[case::max_tokens("max_tokens", "length")]
#[case::pause_turn("pause_turn", "length")]
#[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
#[case::refusal("refusal", "content_filter")]
#[case::guardrail_intervened("guardrail_intervened", "content_filter")]

4
litellm/_lazy_imports_registry.py
View file

@ -1469,11 +1469,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
"AsyncHTTPHandler": ("litellm.llms.custom_httpx.http_handler", "AsyncHTTPHandler"),
"HTTPHandler": ("litellm.llms.custom_httpx.http_handler", "HTTPHandler"),
"get_num_retries_from_retry_policy": (
"litellm.router_utils.get_retry_from_policy",
"litellm.litellm_core_utils.get_retry_from_policy",
"get_num_retries_from_retry_policy",
),
"reset_retry_policy": (
"litellm.router_utils.get_retry_from_policy",
"litellm.litellm_core_utils.get_retry_from_policy",
"reset_retry_policy",
),
"get_coroutine_checker": (

4
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -22,7 +22,7 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
A2AStreamingContext,
)
from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS
from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS, A2A_COMPLETION_BRIDGE_WORKING_MESSAGE
from litellm.interactions.agents.utils import merge_agent_headers
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.utils import ModelResponse
@ -268,7 +268,7 @@ class A2ACompletionBridgeHandler:
ctx=ctx,
state="working",
final=False,
message_text="Processing request...",
message_text=A2A_COMPLETION_BRIDGE_WORKING_MESSAGE,
)
yield working_event

112
litellm/a2a_protocol/main.py
View file

@ -191,6 +191,23 @@ def _set_litellm_params_on_logging_obj(
}
async def _record_send_message_billing(
kwargs: Mapping[str, object],
request: "SendMessageRequest",
response_dict: Mapping[str, object],
litellm_params: Mapping[str, object],
agent_id: str | None,
) -> None:
prompt_tokens, completion_tokens, _ = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
request=request,
response_dict=response_dict,
)
_set_usage_on_logging_obj(kwargs=kwargs, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
if "error" not in response_dict:
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
def _get_a2a_model_info(a2a_client: "A2AClientType", kwargs: Mapping[str, object]) -> str:
"""
Extract agent info and set model/custom_llm_provider for cost tracking.
@ -483,13 +500,21 @@ async def asend_message(
if custom_llm_provider:
if request is None:
raise ValueError("request is required for completion bridge")
return await _send_message_via_completion_bridge(
bridge_response: Final = await _send_message_via_completion_bridge(
request=request,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
litellm_params=litellm_params,
agent_extra_headers=agent_extra_headers,
)
await _record_send_message_billing(
kwargs=kwargs,
request=request,
response_dict=bridge_response.model_dump(mode="json", exclude_none=True),
litellm_params=litellm_params,
agent_id=agent_id,
)
return bridge_response
# Standard A2A client flow
if request is None:
@ -538,30 +563,14 @@ async def asend_message(
# Wrap in LiteLLM response type for _hidden_params support
response: Final = LiteLLMSendMessageResponse.from_a2a_response(a2a_response, request_id=str(request.id))
# Calculate token usage from request and response
response_dict: Final[dict[str, object]] = a2a_response.root.model_dump(mode="json", exclude_none=True)
(
prompt_tokens,
completion_tokens,
_,
) = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
request=request,
response_dict=response_dict,
)
# Set usage on logging obj for standard logging payload
_set_usage_on_logging_obj(
await _record_send_message_billing(
kwargs=kwargs,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
request=request,
response_dict=a2a_response.root.model_dump(mode="json", exclude_none=True),
litellm_params=litellm_params,
agent_id=agent_id,
)
# Merge agent pricing params into the logging obj so cost is calculated
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
# Set agent_id on logging obj for SpendLogs tracking
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
return response
@ -646,6 +655,7 @@ async def asend_message_streaming(
metadata: dict[str, object] | None = None,
proxy_server_request: dict[str, object] | None = None,
agent_extra_headers: dict[str, str] | None = None,
agent_name: str | None = None,
**kwargs: object,
) -> AsyncIterator["SendStreamingMessageResponse | Mapping[str, object]"]:
"""
@ -661,6 +671,7 @@ async def asend_message_streaming(
agent_id: Optional agent ID for tracking in SpendLogs
metadata: Optional metadata dict (contains user_api_key, user_id, team_id, etc.)
proxy_server_request: Optional proxy server request data
agent_name: Agent name logged as a2a_agent/<agent_name> for completion bridge agents
Yields:
SendStreamingMessageResponse chunks from the agent
@ -685,6 +696,7 @@ async def asend_message_streaming(
print(chunk)
```
"""
called_at: Final = datetime.datetime.now()
litellm_params = litellm_params or {}
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
@ -705,20 +717,47 @@ async def asend_message_streaming(
request.params.model_dump(mode="json") if hasattr(request.params, "model_dump") else dict(request.params)
)
async for chunk in A2ACompletionBridgeHandler.handle_streaming(
bridge_stream: Final = A2ACompletionBridgeHandler.handle_streaming(
request_id=str(request.id),
params=params,
litellm_params=litellm_params,
api_base=api_base,
agent_extra_headers=agent_extra_headers,
):
yield chunk
)
caller_logging_obj: Final = kwargs.get("litellm_logging_obj")
bridge_logging_obj: Final = (
caller_logging_obj
if isinstance(caller_logging_obj, Logging)
else _build_streaming_logging_obj(
request=request,
agent_name=agent_name or "unknown",
agent_id=agent_id,
litellm_params=litellm_params,
metadata=metadata,
proxy_server_request=proxy_server_request,
start_time=called_at,
)
)
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
bridge_iterator: Final = A2AStreamingIterator(
stream=bridge_stream,
request=request,
logging_obj=bridge_logging_obj,
agent_name=agent_name or "unknown",
)
try:
async for chunk in bridge_iterator:
yield chunk
except (GeneratorExit, asyncio.CancelledError):
await bridge_iterator.abill_on_disconnect()
raise
return
if request is None:
raise ValueError("request is required")
called_at: Final = datetime.datetime.now()
_raw_logging_obj: Final = kwargs.get("litellm_logging_obj")
logging_obj: Logging | None = _raw_logging_obj if isinstance(_raw_logging_obj, Logging) else None
@ -742,12 +781,12 @@ async def asend_message_streaming(
assert a2a_client is not None
agent_name: Final = _get_a2a_model_info(a2a_client, kwargs)
card_agent_name: Final = _get_a2a_model_info(a2a_client, kwargs)
if logging_obj is None:
logging_obj = _build_streaming_logging_obj(
request=request,
agent_name=agent_name,
agent_name=card_agent_name,
agent_id=agent_id,
litellm_params=litellm_params,
metadata=metadata,
@ -755,7 +794,7 @@ async def asend_message_streaming(
start_time=called_at,
)
verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, agent_name)
verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, card_agent_name)
agent_card: Final = _get_a2a_client_agent_card(a2a_client)
card_url: Final = get_agent_card_url(agent_card) if agent_card else None
@ -766,18 +805,23 @@ async def asend_message_streaming(
agent_card=agent_card,
card_url=card_url,
api_base=api_base,
agent_name=agent_name,
agent_name=card_agent_name,
)
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
async for chunk in A2AStreamingIterator(
url_agent_iterator: Final = A2AStreamingIterator(
stream=stream,
request=request,
logging_obj=logging_obj,
agent_name=agent_name,
):
yield chunk
agent_name=card_agent_name,
)
try:
async for chunk in url_agent_iterator:
yield chunk
except (GeneratorExit, asyncio.CancelledError):
await url_agent_iterator.abill_on_disconnect()
raise
async def create_a2a_client(

2
litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py
View file

@ -515,6 +515,8 @@ class PydanticAITransformation:
"contextId": context_id,
"kind": "artifact-update",
"taskId": task_id,
"append": i > 0,
"lastChunk": is_last_chunk,
"artifact": {
"artifactId": artifact_id,
"parts": [

2
litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py
View file

@ -206,6 +206,8 @@ class WatsonxOrchestrateTransformation:
"contextId": context_id,
"kind": "artifact-update",
"taskId": task_id,
"append": i > 0,
"lastChunk": is_last,
"artifact": {
"artifactId": artifact_id,
"parts": [{"kind": "text", "text": chunk_text}],

89
litellm/a2a_protocol/streaming_iterator.py
View file

@ -3,16 +3,18 @@ A2A Streaming Iterator with token tracking and logging support.
"""
import asyncio
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Final
import anyio
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.constants import A2A_COMPLETION_BRIDGE_WORKING_MESSAGE
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -23,6 +25,23 @@ if TYPE_CHECKING:
_OPAQUE_PAYLOAD: Final = TypeAdapter(object)
def _chunk_payload(chunk: "SendStreamingMessageResponse | Mapping[str, object]") -> Mapping[str, object]:
if isinstance(chunk, Mapping):
return chunk
return chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
def _artifact_update(payload: Mapping[str, object]) -> tuple[str, bool] | None:
result: Final = as_str_mapping(payload.get("result"))
if result is None or result.get("kind") != "artifact-update":
return None
artifact: Final = as_str_mapping(result.get("artifact"))
artifact_id: Final = artifact.get("artifactId") if artifact is not None else None
if not isinstance(artifact_id, str):
return None
return artifact_id, result.get("append") is True
class A2AStreamingIterator:
"""
Async iterator for A2A streaming responses with token tracking.
@ -32,7 +51,7 @@ class A2AStreamingIterator:
def __init__(
self,
stream: AsyncIterator["SendStreamingMessageResponse"],
stream: AsyncIterator["SendStreamingMessageResponse | Mapping[str, object]"],
request: "SendStreamingMessageRequest",
logging_obj: LiteLLMLoggingObj,
agent_name: str = "unknown",
@ -44,14 +63,16 @@ class A2AStreamingIterator:
self.start_time = logging_obj.start_time
# Collect chunks for token counting
self.chunks: list[SendStreamingMessageResponse] = []
self.collected_text_parts: list[str] = []
self.final_chunk: SendStreamingMessageResponse | None = None
self.chunks: list[SendStreamingMessageResponse | Mapping[str, object]] = []
self.latest_message_text = ""
self.artifact_texts: Mapping[str, str] = {}
self.saw_error = False
self.final_chunk: SendStreamingMessageResponse | Mapping[str, object] | None = None
def __aiter__(self):
return self
async def __anext__(self) -> "SendStreamingMessageResponse":
async def __anext__(self) -> "SendStreamingMessageResponse | Mapping[str, object]":
try:
chunk: Final = await self.stream.__anext__()
if self.logging_obj.completion_start_time is None:
@ -59,38 +80,54 @@ class A2AStreamingIterator:
# Store chunk
self.chunks.append(chunk)
payload: Final = _chunk_payload(chunk)
self.saw_error = self.saw_error or payload.get("error") is not None
# Extract text from chunk for token counting
self._collect_text_from_chunk(chunk)
self._collect_text_from_chunk(payload)
# Check if this is the final chunk (completed status)
if self._is_completed_chunk(chunk):
if self._is_completed_chunk(payload):
self.final_chunk = chunk
return chunk
except StopAsyncIteration:
# Stream ended - handle logging
if self.final_chunk is None and self.chunks:
self.final_chunk = self.chunks[-1]
await self._handle_stream_complete()
raise
def _collect_text_from_chunk(self, chunk: "SendStreamingMessageResponse") -> None:
"""Extract text from a streaming chunk and add to collected parts."""
async def abill_on_disconnect(self) -> None:
if not self.chunks:
return
with anyio.CancelScope(shield=True):
await self._handle_stream_complete()
def _collect_text_from_chunk(self, payload: Mapping[str, object]) -> None:
"""Record the reply text a streaming chunk carries."""
try:
chunk_dict: Final = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
text: Final = A2ARequestUtils.extract_text_from_response(chunk_dict)
if text:
self.collected_text_parts.append(text)
text: Final = A2ARequestUtils.extract_text_from_stream_event(payload)
if not text:
return
artifact_update: Final = _artifact_update(payload)
if artifact_update is None and text == A2A_COMPLETION_BRIDGE_WORKING_MESSAGE:
return
if artifact_update is None:
self.latest_message_text = text
return
artifact_id, append = artifact_update
artifact_text: Final = self.artifact_texts.get(artifact_id, "") + text if append else text
self.artifact_texts = {**self.artifact_texts, artifact_id: artifact_text}
except Exception:
verbose_logger.debug("Failed to extract text from A2A streaming chunk")
def _is_completed_chunk(self, chunk: "SendStreamingMessageResponse") -> bool:
def _reply_text(self) -> str:
artifact_text: Final = " ".join(text for text in self.artifact_texts.values() if text)
return artifact_text or self.latest_message_text
def _is_completed_chunk(self, payload: Mapping[str, object]) -> bool:
"""Check if chunk indicates stream completion."""
try:
chunk_dict: Final = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
result: Final = chunk_dict.get("result", {})
result: Final = payload.get("result", {})
if isinstance(result, dict):
status: Final = result.get("status", {})
if isinstance(status, dict):
@ -101,6 +138,8 @@ class A2AStreamingIterator:
async def _handle_stream_complete(self) -> None:
"""Handle logging and token counting when stream completes."""
if self.final_chunk is None and self.chunks:
self.final_chunk = self.chunks[-1]
try:
end_time: Final = datetime.now()
@ -109,9 +148,7 @@ class A2AStreamingIterator:
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
prompt_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(input_text)
# Use the last (most complete) text from chunks
output_text: Final = self.collected_text_parts[-1] if self.collected_text_parts else ""
completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(output_text)
completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(self._reply_text())
total_tokens: Final = prompt_tokens + completion_tokens
@ -128,7 +165,7 @@ class A2AStreamingIterator:
self.logging_obj.model_call_details["stream"] = False
# Calculate cost using A2ACostCalculator
response_cost: Final = A2ACostCalculator.calculate_a2a_cost(self.logging_obj)
response_cost: Final = 0.0 if self.saw_error else A2ACostCalculator.calculate_a2a_cost(self.logging_obj)
self.logging_obj.model_call_details["response_cost"] = response_cost
# Build result for logging
@ -167,9 +204,7 @@ class A2AStreamingIterator:
# Add final chunk result if available
if self.final_chunk:
try:
chunk_dict: Final = as_str_mapping(
_OPAQUE_PAYLOAD.validate_python(self.final_chunk.model_dump(mode="json", exclude_none=True))
)
chunk_dict: Final = as_str_mapping(_OPAQUE_PAYLOAD.validate_python(_chunk_payload(self.final_chunk)))
if chunk_dict is not None:
result["result"] = chunk_dict.get("result", {})
except Exception:

50
litellm/a2a_protocol/utils.py
View file

@ -6,13 +6,17 @@ import hashlib
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Protocol
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.core_helpers import as_str_mapping, has_attributes
if TYPE_CHECKING:
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
_OBJECT_ITEMS: Final = TypeAdapter(tuple[object, ...])
class _SupportsModelDump(Protocol):
def model_dump(self, *, mode: str) -> object: ...
@ -34,6 +38,9 @@ class A2ARequestUtils:
"""
if message is None:
return ""
# a2a-sdk Part RootModels keep kind/text on part.root, so read pydantic messages as dicts
if has_attributes(message, _SupportsModelDump, "model_dump"):
return A2ARequestUtils.extract_text_from_message(message.model_dump(mode="json"))
# Handle both dict and object access
if isinstance(message, dict):
@ -63,8 +70,8 @@ class A2ARequestUtils:
Returns:
Text from response message parts
"""
result: Final = response_dict.get("result", {})
if not isinstance(result, dict):
result: Final = as_str_mapping(response_dict.get("result"))
if result is None:
return ""
# Direct message format (A2A spec): detect by explicit kind tag only.
@ -72,10 +79,36 @@ class A2ARequestUtils:
# type that happens to include a "parts" field.
if result.get("kind") == "message":
return A2ARequestUtils.extract_text_from_message(result)
if result.get("kind") == "task":
return A2ARequestUtils.extract_text_from_task(result)
message: Final = result.get("message", {})
return A2ARequestUtils.extract_text_from_message(message)
@staticmethod
def extract_text_from_task(task: Mapping[str, object]) -> str:
artifacts: Final = task.get("artifacts")
artifact_items: Final = _OBJECT_ITEMS.validate_python(artifacts) if isinstance(artifacts, list) else ()
artifact_texts: Final = tuple(
A2ARequestUtils.extract_text_from_message(artifact) for artifact in artifact_items
)
if any(artifact_texts):
return " ".join(text for text in artifact_texts if text)
status: Final = as_str_mapping(task.get("status"))
return A2ARequestUtils.extract_text_from_message(status.get("message")) if status is not None else ""
@staticmethod
def extract_text_from_stream_event(event: Mapping[str, object]) -> str:
result: Final = as_str_mapping(event.get("result"))
if result is None:
return ""
if result.get("kind") == "artifact-update":
return A2ARequestUtils.extract_text_from_message(result.get("artifact"))
if result.get("kind") == "status-update":
status: Final = as_str_mapping(result.get("status"))
return A2ARequestUtils.extract_text_from_message(status.get("message")) if status is not None else ""
return A2ARequestUtils.extract_text_from_response(event)
@staticmethod
def get_input_message_from_request(
request: "SendMessageRequest | SendStreamingMessageRequest",
@ -128,17 +161,8 @@ class A2ARequestUtils:
Returns:
Tuple of (prompt_tokens, completion_tokens, total_tokens)
"""
# Count input tokens. Dump the message to a dict first so extraction hits
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
input_text: Final = A2ARequestUtils.extract_text_from_message(
input_message.model_dump(mode="json")
if has_attributes(input_message, _SupportsModelDump, "model_dump")
else input_message
)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
prompt_tokens: Final = A2ARequestUtils.count_tokens(A2ARequestUtils.extract_text_from_message(input_message))
# Count output tokens
output_text: Final = A2ARequestUtils.extract_text_from_response(response_dict)

13
litellm/caching/caching.py
View file

@ -25,7 +25,11 @@ from litellm._internal_context import current_service_target, service_target
from litellm._logging import verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.integrations.otel.runtime import phase_span
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
from litellm.litellm_core_utils.model_param_helper import (
ModelParamHelper,
embeds_thought_signature_in_tool_call_id,
peek_reasoning_summary_aliases,
)
from litellm.types.caching import *
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
@ -421,7 +425,12 @@ class Cache:
reasoning_summary_material: Final = (
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
tool_call_id_shape_material: Final = (
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
)
hashed_cache_key = Cache._get_hashed_cache_key(
cache_key + reasoning_summary_material + tool_call_id_shape_material
)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",

8
litellm/constants.py
View file

@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# in-memory stand-in handed to provider converters for redacted arguments; never stored
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
@ -634,6 +636,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
@ -902,6 +906,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
A2A_COMPLETION_BRIDGE_WORKING_MESSAGE: Final = "Processing request..."
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
"openai",
@ -1162,6 +1167,7 @@ openai_compatible_providers: Final[list[str]] = [
"cortecs",
"scx-ai",
"prism",
"hostinger_router",
"sail",
"reka",
"streamlake",
@ -1685,6 +1691,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
@ -2046,6 +2053,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = get_env_int("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
AUTH_OBJECTS_TARGET: Final = "auth_objects"
HEALTH_CHECKS_TARGET: Final = "health_checks"
DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = get_env_int("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)
SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600
SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30

2
litellm/cost_calculator.py
View file

@ -3319,7 +3319,7 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
)
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"})
class _ResponsesWsEventResponse(LiteLLMBaseModel):

160
litellm/decisions/call.py
View file

@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from functools import partial
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing import Annotated, Final, Literal, TypeAlias
import httpx
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError
from pydantic_core import to_jsonable_python
from typing_extensions import assert_never
@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
DecisionsResponseValidationError,
catalog_mode,
image_media_type,
image_param,
input_images,
@ -61,7 +62,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.llms.openai import ChatCompletionUserMessage
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
@ -257,19 +258,105 @@ def _request_safety_identifier(
return None
def _fields_without_unsupported_media(
fields: RequestFields,
def _bool_or_none(value: object) -> bool | None:
return value if isinstance(value, bool) else None
def _str_or_none(value: object) -> str | None:
return value if isinstance(value, str) else None
_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)]
class _DeploymentCapabilities(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None
supports_box_questions: _DeploymentFlag = None
supports_audio_input: _DeploymentFlag = None
supports_video_input: _DeploymentFlag = None
def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None:
if flag is None:
return None
mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider)
return flag if mode is None or is_decisions_model_mode(mode) else None
@dataclass(frozen=True, slots=True)
class _ResolvedSupport:
box_questions: bool
audio: bool
videos: bool
def _takes_audio(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_audio_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
)
def _takes_videos(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_video_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
)
def deployment_refuses_media(
model: str,
custom_llm_provider: str,
model_info: Mapping[str, object],
call_kwargs: Mapping[str, object],
*,
audio: bool,
videos: bool,
) -> bool:
if _drops_params(call_kwargs):
return False
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
canonical_model: Final = provider_config.canonical_model(model)
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
)
def _resolved_support(
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
kwargs: Mapping[str, object],
fields: RequestFields,
) -> _ResolvedSupport:
deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {})
return _ResolvedSupport(
box_questions=provider_config.supports_box_questions(
model, custom_llm_provider, headers, deployment.supports_box_questions
),
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
)
def _fields_without_unsupported_media(
fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object]
) -> RequestFields:
if not _drops_params(kwargs):
return fields
return replace(
fields,
audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
audio=fields.audio if support.audio else None,
videos=fields.videos if support.videos else None,
)
@ -292,10 +379,8 @@ class _MalformedImages:
def _unsupported_params(
ir_request: DecisionsIRRequest,
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
supports_box_questions: bool,
support: _ResolvedSupport,
) -> tuple[str, ...]:
refuses_safety_identifier: Final = (
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
@ -304,13 +389,9 @@ def _unsupported_params(
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
)
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
model, custom_llm_provider
)
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
model, custom_llm_provider
)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
refuses_audio: Final = bool(ir_request.audio) and not support.audio
refuses_videos: Final = bool(ir_request.videos) and not support.videos
return (
*(("safety_identifier",) if refuses_safety_identifier else ()),
*(() if refused_image_param is None else (refused_image_param,)),
@ -333,7 +414,7 @@ def _without_unsupported_params(
ir_request: DecisionsIRRequest,
provider_config: BaseDecisionsConfig,
headers: Mapping[str, str],
supports_box_questions: bool,
support: _ResolvedSupport,
) -> DecisionsIRRequest:
return replace(
ir_request,
@ -345,9 +426,7 @@ def _without_unsupported_params(
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
questions=(
ir_request.questions
if supports_box_questions
else tuple(filter(_is_not_box_question, ir_request.questions))
ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions))
),
)
@ -368,39 +447,20 @@ def _without_malformed_images(
return well_formed
class _DeploymentBoxQuestions(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
supports_box_questions: StrictBool | None = None
def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None:
try:
return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions
except ValidationError:
return None
def _provider_ir_request(
ir_request: DecisionsIRRequest,
*,
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
support: _ResolvedSupport,
kwargs: Mapping[str, object],
) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
supports_box_questions: Final = provider_config.supports_box_questions(
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
)
unsupported: Final = _unsupported_params(
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
)
unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
if not unsupported:
return ir_request
if not _drops_params(kwargs):
return _UnsupportedParams(names=unsupported)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
if _leaves_no_input(supported.input):
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
if not supported.questions:
@ -479,10 +539,11 @@ def prepare_call(
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
try:
headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs)
support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields)
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
request: Final = _fields_without_unsupported_media(
fields, provider_config, canonical_model, provider, kwargs
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate(
safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)
)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Decisions request: {error}",
@ -523,12 +584,7 @@ def prepare_call(
llm_provider=provider,
)
ir_request: Final = _provider_ir_request(
caller_ir_request,
provider_config=provider_config,
model=canonical_model,
custom_llm_provider=provider,
headers=headers,
kwargs=kwargs,
caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
)
if isinstance(ir_request, _UnsupportedParams):
raise litellm.UnsupportedParamsError(

210
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -14,7 +14,7 @@ import itertools
import os
import re
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
return None
_MessageT: Final = TypeVar("_MessageT")
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
itertools.chain.from_iterable(
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
)
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
if fields.get("cache_control") is None:
return False
return (
not isinstance(fields.get("content"), list)
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
)
def cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(_message_mark_reaches_provider(fields)) + block_marks
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def dropped_cache_control(_: object) -> None:
return None
def _first_marked_index(value: object) -> int | None:
if not isinstance(value, list):
return None
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
_MarkSlot: TypeAlias = tuple[str, int]
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
value: Final = fields.get(key)
if not isinstance(value, list):
return ()
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
content_slots: Final = _block_mark_slots(fields, "content")
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
if not _message_mark_reaches_provider(fields):
return content_slots + tool_call_slots
if isinstance(fields.get("content"), list):
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
return value
return [
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
]
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
if dropped == 0 or not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
_MessageT,
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
if _MESSAGE_MARK_SLOT in dropped_slots
else blocks_unmarked,
)
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
content: Final = fields.get("content")
if fields.get("role") != "tool" or not isinstance(content, list):
return message
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
_MessageT,
{
**fields,
"content": [
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
],
},
)
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
kept: Final = min(marks, remaining)
return remaining - kept, marks - kept
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
if marks > remaining:
return remaining, marks
return remaining - marks, 0
def _messages_within_budget(
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = itertools.accumulate(
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
)
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
)
return tuple(reversed(kept_newest_first))
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
def whole_messages_within_cache_control_limit(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
def _part_with_cache_control_renamed(part: object) -> object:
if not _accepts_prompt_cache_breakpoint(part):
return part
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
llm_router=None,
)
return None
def messages_within_cache_control_cap(
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
) -> Sequence[_MessageT]:
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
)
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:
verbose_logger.warning(
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
"provider accepts at most %d cache_control blocks per request",
dropped,
MAX_CACHE_CONTROL_BLOCKS,
)
return kept

8
litellm/integrations/prometheus.py
View file

@ -66,7 +66,7 @@ from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.table_repositories import EndUserRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
from litellm.router_api import deployment_metadata
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.integrations.prometheus import *
from litellm.types.integrations.prometheus import (
@ -3022,8 +3022,8 @@ class PrometheusLogger(CustomLogger):
).get("custom_llm_provider")
if isinstance(provider, str) and provider:
return provider
deployment_metadata: Final = find_deployment_metadata(litellm_params)
return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None)
picked_deployment: Final = deployment_metadata(litellm_params)
return _deployment_model_provider(picked_deployment.get("deployment") if picked_deployment else None)
@staticmethod
def _extract_api_provider_from_exception(exception: Exception) -> str | None:
@ -5209,7 +5209,7 @@ def _tag_matches_wildcard_configured_pattern(tags: Sequence[str], configured_tag
"""
import re
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
pattern_router: Final = PatternMatchRouter()
regex_pattern: Final = pattern_router.pattern_to_regex(configured_tag)

0
litellm/router_utils/access_windows.py → litellm/litellm_core_utils/access_windows.py
View file

0
litellm/router_utils/add_retry_fallback_headers.py → litellm/litellm_core_utils/add_retry_fallback_headers.py
View file

0
litellm/router_utils/clientside_credential_handler.py → litellm/litellm_core_utils/clientside_credential_handler.py
View file

1
litellm/litellm_core_utils/core_helpers.py
View file

@ -311,6 +311,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
"tool_use": "tool_calls",
"refusal": "content_filter",
"compaction": "length",
"pause_turn": "length",
"model_context_window_exceeded": "length",
# Cohere
"COMPLETE": "stop",

0
litellm/router_utils/deployment_pricing.py → litellm/litellm_core_utils/deployment_pricing.py
View file

6
litellm/litellm_core_utils/fallback_utils.py
View file

@ -3,13 +3,13 @@ from typing import Final
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
)
from litellm.litellm_core_utils.core_helpers import (
filter_internal_params,
safe_deep_copy,
)
from litellm.router_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
)
from .asyncify import run_async_function

3
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -84,6 +84,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_host",
"github_copilot_user_session",
"fireworks_forward_user_id",
"thought_signature_in_tool_call_id",
"responses_api_version",
"allowed_openai_params",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
@ -99,6 +100,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
# Backward-compatible alias for existing imports/tests.
_OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS
SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"}
_CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions)
_CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions))
_MAX_SHOWN_INT_BITS: Final = 64

0
litellm/router_utils/get_retry_from_policy.py → litellm/litellm_core_utils/get_retry_from_policy.py
View file

2
litellm/litellm_core_utils/health_check_helpers.py
View file

@ -119,7 +119,7 @@ def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segmen
def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
provider_prefix: Final = f"{custom_llm_provider}/"
segments: Final = wildcard.removeprefix(provider_prefix).split("/")

3
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking:
usage=usage, response_object=response_object
)
response_model: Final = _output_item_field(response_object, "model")
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
result: Final = get_cost_for_web_search_request(
custom_llm_provider=resolved_provider,
usage=resolved_usage,
model_info=model_info,
response_model=response_model if isinstance(response_model, str) else None,
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
)
if result is not None:
return result

52
litellm/litellm_core_utils/logging_utils.py
View file

@ -3,13 +3,15 @@ import functools
import inspect
import re
import time
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
from litellm._logging import format_base64_size, verbose_logger
from litellm.constants import (
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS,
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS,
MAX_BASE64_LENGTH_FOR_LOGGING,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
@ -44,6 +46,8 @@ Helper utils used for logging callbacks
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
# Captures: group(1)=mime_type, group(2)=base64_payload
_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)")
_DATA_URI_PREFIX: Final = "data:"
_DATA_URI_BASE64_MARKER: Final = ";base64,"
# Maximum nesting depth for _truncate_base64_in_value to guard against
# pathological payloads. OpenAI message format is typically 3-4 levels deep.
@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str:
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
def _may_contain_base64_data_uri(value: str) -> bool:
return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value
def _truncate_base64_in_string(value: str) -> str:
"""Replace long base64 data-URI payloads in a string with a size placeholder."""
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value):
return value
return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
def _truncate_base64_in_value(
value: str | dict[str, object] | list[object] | None,
truncate_string: Callable[[str], str],
) -> str | dict[str, object] | list[object] | None:
"""Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
@ -78,13 +87,13 @@ def _truncate_base64_in_value(
# Stack entries: (source_value, depth, parent_container, key_or_index)
# We mutate *copies* of dicts/lists in-place via parent references.
if isinstance(value, str):
return _truncate_base64_in_string(value)
return truncate_string(value)
if not isinstance(value, (dict, list)):
return value
# Shallow-copy the root so we don't mutate the caller's data.
root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value)
stack: Final[list] = [(root, 0)]
stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack
while stack:
container, depth = stack.pop()
@ -93,19 +102,19 @@ def _truncate_base64_in_value(
if isinstance(container, dict):
for k, v in container.items():
if isinstance(v, str):
container[k] = _truncate_base64_in_string(v)
container[k] = truncate_string(v)
elif isinstance(v, dict):
copy: dict | list = {ck: cv for ck, cv in v.items()}
copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()}
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(v, list):
copy = list(v)
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(container, list):
else:
for i, v in enumerate(container):
if isinstance(v, str):
container[i] = _truncate_base64_in_string(v)
container[i] = truncate_string(v)
elif isinstance(v, dict):
copy = {ck: cv for ck, cv in v.items()}
container[i] = copy
@ -127,8 +136,15 @@ def truncate_base64_in_messages(
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
return _copy_messages(messages, _truncate_base64_in_string)
def _copy_messages(
messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages
truncate_string: Callable[[str], str],
) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages
try:
return _truncate_base64_in_value(messages)
return _truncate_base64_in_value(messages, truncate_string)
except Exception as e:
verbose_logger.debug("Failed to truncate base64 in messages: %s", e)
return messages
@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
pass
def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool:
return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any(
_may_contain_base64_data_uri(leaf) for leaf in leaves
)
async def truncate_base64_in_messages_async(
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
"""
Same result as truncate_base64_in_messages, but payloads whose string content
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
thread so the regex pass over multi-MB base64 images does not block the event loop.
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread,
so neither the regex pass over multi-MB base64 images nor the copy of a long
history blocks the event loop. A few strings with no data URI, under
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since
checking them for data URIs and copying them costs less than the thread hop.
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
leaves: Final = tuple(_iter_string_leaves(messages))
total_chars: Final = sum(len(leaf) for leaf in leaves)
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
return truncate_base64_in_messages(messages)
if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves):
return _copy_messages(messages, lambda text: text)
return await asyncio.to_thread(truncate_base64_in_messages, messages)

11
litellm/litellm_core_utils/model_param_helper.py
View file

@ -17,6 +17,7 @@ from openai.types.responses.response_create_params import (
ResponseCreateParamsNonStreaming,
ResponseCreateParamsStreaming,
)
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.types.llms.anthropic import AnthropicMessagesRequest
@ -41,6 +42,16 @@ def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> obj
return None
_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None)
def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool:
try:
return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False
except ValidationError:
return True
class ModelParamHelper:
# Cached at class level — deterministic set built from static OpenAI type annotations
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()

0
litellm/router_utils/pattern_match_deployments.py → litellm/litellm_core_utils/pattern_match_deployments.py
View file

17
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
return [{"input": str(messages)}]
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
texts: Final = convert_content_list_to_str(message=message)
if not texts:
return message
return cast( # cast-ok: the same message with its content flattened to one string
AllMessageValues, {**message, "content": texts}
)
def handle_messages_with_content_list_to_str_conversion(
messages: list[AllMessageValues],
) -> list[AllMessageValues]:
"""
Handles messages with content list conversion
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
"""
for message in messages:
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
return messages
return [_message_with_str_content(message) for message in messages]
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:

179
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -26,6 +26,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_messages,
find_anthropic_server_tool_result,
is_anthropic_server_tool_use,
is_anthropic_tool_search_use,
message_field,
parts_of,
@ -1252,6 +1253,22 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str
_encode_tool_call_id_with_signature = encode_tool_call_id_with_signature
def _tool_call_id_prefix(tool_call_id: object) -> str | None:
return tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] if isinstance(tool_call_id, str) else None
def _called_tool_name(tool_call_id: str, tool_calls: Sequence[Mapping[str, object]]) -> str | None:
exact_matches: Final = tuple(call for call in tool_calls if call.get("id") == tool_call_id)
prefix: Final = _tool_call_id_prefix(tool_call_id)
prefix_matches: Final = tuple(
call for call in tool_calls if prefix and _tool_call_id_prefix(call.get("id")) == prefix
)
matches: Final = exact_matches or (prefix_matches if len(prefix_matches) == 1 else ())
function: Final = matches[0].get("function") if matches else None
name: Final = function.get("name") if isinstance(function, Mapping) else None
return name if isinstance(name, str) else None
def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
"""Extract thought signature from tool call's provider_specific_fields.
@ -1691,16 +1708,14 @@ def convert_to_gemini_tool_call_result(
)
) is not None:
media_parts.append(media_part)
name: str | None = message.get("name", "")
# Recover name from last message with tool calls
if last_message_with_tool_calls:
tools: Final = last_message_with_tool_calls.get("tool_calls", [])
msg_tool_call_id: Final = message.get("tool_call_id", None)
for tool in tools:
prev_tool_call_id = tool.get("id", None)
if msg_tool_call_id and prev_tool_call_id and msg_tool_call_id == prev_tool_call_id:
name = tool.get("function", {}).get("name", "")
msg_tool_call_id: Final = message.get("tool_call_id", None)
called_tool_name: Final = (
_called_tool_name(msg_tool_call_id, last_message_with_tool_calls.get("tool_calls") or ())
if last_message_with_tool_calls and isinstance(msg_tool_call_id, str) and msg_tool_call_id
else None
)
name: Final[str | None] = called_tool_name if called_tool_name is not None else message.get("name", "")
# Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix).
gemini_call_id: str | None = None
@ -2059,6 +2074,54 @@ def _placed_and_paired(
_AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, Any]
def _is_server_tool_result(block: Mapping[str, object]) -> bool:
block_type: Final = block.get("type")
return isinstance(block_type, str) and block_type.endswith("_tool_result")
def _split_unrun_server_tool_uses(
groups: Sequence[Sequence[_AnthropicToolInvokeItem]],
) -> tuple[tuple[Sequence[_AnthropicToolInvokeItem], ...], tuple[_AnthropicToolInvokeItem, ...]]:
completed: Final = tuple(group for group in groups if any(_is_server_tool_result(block) for block in group))
unrun: Final = tuple(group[0] for group in groups if not any(_is_server_tool_result(block) for block in group))
return completed, unrun
def _block_id(block: Mapping[str, object]) -> object:
return block.get("id")
def _unrun_server_tool_use_ids(blocks: Sequence[Mapping[str, object]]) -> frozenset[object]:
result_ids: Final = frozenset(block.get("tool_use_id") for block in blocks if _is_server_tool_result(block))
return frozenset(
_block_id(block)
for block in blocks
if block.get("type") == "server_tool_use" and _block_id(block) not in result_ids
)
def _last_block_id_among(blocks: Sequence[Mapping[str, object]], ids: frozenset[object]) -> object:
return next(block_id for block in reversed(blocks) if (block_id := _block_id(block)) in ids)
def _paused_turn_replay(
tool_invokes: Sequence[_AnthropicToolInvokeItem],
thinking_blocks: Sequence[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
*,
is_final_message: bool,
) -> tuple[
tuple[_AnthropicToolInvokeItem, ...],
tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...],
]:
unrun_ids: Final = _unrun_server_tool_use_ids(tool_invokes)
if not unrun_ids:
return tuple(tool_invokes), tuple(thinking_blocks)
dropped_ids: Final = (
unrun_ids - frozenset((_last_block_id_among(tool_invokes, unrun_ids),)) if is_final_message else unrun_ids
)
return tuple(block for block in tool_invokes if _block_id(block) not in dropped_ids), ()
def convert_to_anthropic_tool_invoke(
tool_calls: list[ChatCompletionAssistantToolCall],
web_search_results: Sequence[object] | None = None,
@ -2141,6 +2204,15 @@ def convert_to_anthropic_tool_invoke(
anthropic_tool_invoke.append(server_tool_result)
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
continue
elif is_anthropic_server_tool_use(tool_id) and tool_id not in answered_tool_ids:
anthropic_tool_invoke.append(
{
"type": "server_tool_use",
"id": tool_id,
"name": tool_name,
"input": tool_input,
}
)
else:
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
@ -3109,6 +3181,30 @@ def anthropic_messages_pt(
_has_server_tool_calls = True
break
# Get web_search_results and tool_results from provider_specific_fields
# for server_tool_use reconstruction.
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields: dict[str, Any] = (
cast(dict[str, Any], _provider_specific_fields_raw)
if isinstance(_provider_specific_fields_raw, dict)
else {}
)
tool_invoke_results = (
convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_provider_specific_fields.get("web_search_results"),
tool_results=_provider_specific_fields.get("tool_results"),
answered_tool_ids=client_answered_tool_ids,
)
if assistant_tool_calls is not None
else ()
)
turn_paused = bool(_unrun_server_tool_use_ids(tool_invoke_results))
replayed_tool_invokes, replayed_thinking_blocks = _paused_turn_replay(
tool_invoke_results, thinking_blocks or (), is_final_message=msg_i == len(messages) - 1
)
if (
thinking_blocks is not None
and _has_server_tool_calls
@ -3121,26 +3217,12 @@ def anthropic_messages_pt(
# We must preserve this interleaved order because Anthropic
# verifies thinking block signatures based on position.
# Build the tool call groups (server_tool_use + its result)
_provider_specific_fields_raw_tc = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields_tc: dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw_tc, dict):
_provider_specific_fields_tc = cast(dict[str, Any], _provider_specific_fields_raw_tc)
_web_search_results_tc = _provider_specific_fields_tc.get("web_search_results")
_tool_results_tc = _provider_specific_fields_tc.get("tool_results")
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results_tc,
tool_results=_tool_results_tc,
answered_tool_ids=client_answered_tool_ids,
)
# Group tool invoke results into (server_tool_use, result) pairs
# and separate regular tool_use blocks
server_tool_groups: list[list[_AnthropicToolInvokeItem]] = []
regular_tool_uses: list[_AnthropicToolInvokeItem] = []
_current_group: list[_AnthropicToolInvokeItem] = []
for item in tool_invoke_results:
for item in replayed_tool_invokes:
item_type = item.get("type", "") if isinstance(item, dict) else getattr(item, "type", "")
if item_type == "server_tool_use":
if _current_group:
@ -3154,6 +3236,9 @@ def anthropic_messages_pt(
_current_group.append(item)
if _current_group:
server_tool_groups.append(_current_group)
completed_server_tool_groups, pending_server_tool_uses = _split_unrun_server_tool_uses(
server_tool_groups
)
# Build the text block if content is a non-empty string
text_element = None
@ -3177,15 +3262,15 @@ def anthropic_messages_pt(
# Any remaining groups (after all thinking blocks) go after.
tb_idx = 0
grp_idx = 0
num_tb = len(thinking_blocks) if thinking_blocks else 0
num_grp = len(server_tool_groups)
num_tb = len(replayed_thinking_blocks)
num_grp = len(completed_server_tool_groups)
while tb_idx < num_tb or grp_idx < num_grp:
if tb_idx < num_tb and grp_idx < num_grp:
# Emit thinking block then its tool group
assistant_content.append(thinking_blocks[tb_idx])
assistant_content.append(replayed_thinking_blocks[tb_idx])
tb_idx += 1
for block in server_tool_groups[grp_idx]:
for block in completed_server_tool_groups[grp_idx]:
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3195,11 +3280,11 @@ def anthropic_messages_pt(
grp_idx += 1
elif tb_idx < num_tb:
# More thinking blocks than tool groups - emit before text
assistant_content.append(thinking_blocks[tb_idx])
assistant_content.append(replayed_thinking_blocks[tb_idx])
tb_idx += 1
else:
# More tool groups than thinking blocks - emit remaining
for block in server_tool_groups[grp_idx]:
for block in completed_server_tool_groups[grp_idx]:
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3212,8 +3297,8 @@ def anthropic_messages_pt(
if text_element is not None:
assistant_content.append(text_element)
# Add regular (non-server) tool calls at the end
for item in regular_tool_uses:
# Add regular tool calls, then a paused turn's unrun server tool call, at the end
for item in (*regular_tool_uses, *pending_server_tool_uses):
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3233,18 +3318,18 @@ def anthropic_messages_pt(
)
_content_list = assistant_content_block.get("content") if _content_is_list else None
if (
thinking_blocks is not None and not _content_is_list
): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
assistant_content.extend(thinking_blocks)
if not _content_is_list: # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
assistant_content.extend(replayed_thinking_blocks)
if _content_is_list and _content_list is not None:
for m in _with_missing_thinking_blocks(_content_list, thinking_blocks or ()):
for m in _with_missing_thinking_blocks(_content_list, replayed_thinking_blocks):
if not isinstance(m, dict):
continue
# handle thinking blocks
if (thinking_block := _as_thinking_block(m)) is not None:
assistant_content.extend(
_replayable_thinking_blocks([thinking_block], drop_params=drop_params)
()
if turn_paused
else _replayable_thinking_blocks([thinking_block], drop_params=drop_params)
)
continue
text_block = cast(str, m.get("text", ""))
@ -3284,25 +3369,9 @@ def anthropic_messages_pt(
assistant_content.append(_anthropic_text_content_element)
if assistant_tool_calls is not None: # support assistant tool invoke conversion
# Get web_search_results and tool_results from provider_specific_fields
# for server_tool_use reconstruction.
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields: dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw, dict):
_provider_specific_fields = cast(dict[str, Any], _provider_specific_fields_raw)
_web_search_results = _provider_specific_fields.get("web_search_results")
_tool_results = _provider_specific_fields.get("tool_results")
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results,
tool_results=_tool_results,
answered_tool_ids=client_answered_tool_ids,
)
# Prevent "tool_use ids must be unique" errors by filtering duplicates
# This can happen when merging history that already contains the tool calls
for item in tool_invoke_results:
for item in replayed_tool_invokes:
# tool_use items are typically dicts, but handle objects just in case
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)

4
litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py
View file

@ -286,6 +286,10 @@ def _separate_thinking_blocks_render(message: object) -> bool:
return any(_thinking_block_renders(block) for block in parts_of(message_field(message, "thinking_blocks")))
def is_anthropic_server_tool_use(tool_id: str) -> bool:
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES

2
litellm/router_utils/ptu_shares.py → litellm/litellm_core_utils/ptu_shares.py
View file

@ -12,13 +12,13 @@ from datetime import date
from typing import Final, Generic, TypeAlias, TypeVar
from litellm.litellm_core_utils.ptu_pricing import parsed_ptu_shares, ptu_terms
from litellm.litellm_core_utils.routing_common_utils import get_request_team_id, team_may_use_deployment
from litellm.llms.azure.ptu_capacity import (
PTUCapacity,
deployment_ptu_capacity,
is_azure_deployment,
is_str_keyed_mapping,
)
from litellm.router_utils.common_utils import get_request_team_id, team_may_use_deployment
_DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object])
WildcardRoute: TypeAlias = Callable[[str], Sequence[Mapping[str, object]]]

0
litellm/router_utils/reasoning_effort_capability.py → litellm/litellm_core_utils/reasoning_effort_capability.py
View file

0
litellm/router_utils/common_utils.py → litellm/litellm_core_utils/routing_common_utils.py
View file

0
litellm/router_utils/routing_groups.py → litellm/litellm_core_utils/routing_groups.py
View file

19
litellm/litellm_core_utils/streaming_chunk_builder_utils.py
View file

@ -1509,11 +1509,20 @@ def _delta_reasoning_items(delta: object) -> Sequence[object]:
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
def _chunk_first_delta(chunk: object) -> object:
choices: Final = _field_of(chunk, "choices")
if not isinstance(choices, list) or not choices:
return ()
return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta")) # cast-ok: untyped chunk
return None
return _field_of(cast(Sequence[object], choices)[0], "delta") # cast-ok: untyped chunk
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
return _delta_reasoning_items(_chunk_first_delta(chunk))
def _chunk_refusal(chunk: object) -> str:
refusal: Final = _field_of(_chunk_first_delta(chunk), "refusal")
return refusal if isinstance(refusal, str) else ""
_NON_TEXT_DELTA_FIELDS: Final = (
@ -1526,6 +1535,7 @@ _NON_TEXT_DELTA_FIELDS: Final = (
"audio",
"images",
"provider_specific_fields",
"refusal",
)
@ -1674,6 +1684,9 @@ def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> It
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
combined_refusal: Final = "".join(_chunk_refusal(chunk) for chunk in typed_chunks)
if combined_refusal:
yield "refusal", combined_refusal
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
if reasoning_items:
yield "reasoning_items", list(reasoning_items)

2
litellm/litellm_core_utils/token_counter.py
View file

@ -56,9 +56,9 @@ from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEnc
if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
from litellm.router_utils.deployment_pricing import deployment_pricing_models
from litellm.rust_bridge.tokenizer import get_encoding
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,

47
litellm/llms/__init__.py
View file

@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
from . import *
if TYPE_CHECKING:
from collections.abc import Callable
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
)
@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request(
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
def _with_served_model_search_pricing(
model_info: "ModelInfo",
response_model: str | None,
served_model_provider: str,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
) -> "ModelInfo":
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
return model_info
served_model_info: Final = (
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
)
if served_model_info is None:
return model_info
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
def get_cost_for_web_search_request(
custom_llm_provider: str,
usage: "Usage",
model_info: "ModelInfo",
response_model: str | None = None,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
) -> float | None:
"""
Get the cost for a web search request for a given model.
@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
custom_llm_provider: The custom LLM provider.
usage: The usage object.
model_info: The model info.
response_model: The model the provider response names.
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
"""
if custom_llm_provider == "gemini":
from .gemini.cost_calculator import cost_per_web_search_request
@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
elif custom_llm_provider == "anthropic":
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
usage=usage,
)
elif custom_llm_provider.startswith("vertex_ai"):
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
model_key: Final[str] = model_info.get("key", "") if model_info else ""
if "claude" in model_key.lower():
if "claude" in (response_model or model_key).lower():
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(
model_info, response_model, "vertex_ai", lookup_model_info
),
usage=usage,
)
from .vertex_ai.gemini.cost_calculator import (
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
return None
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
usage=usage,
)
elif custom_llm_provider == "perplexity":
# Perplexity handles search costs internally in its own cost calculator
# Return 0.0 to indicate costs are already accounted for

28
litellm/llms/anthropic/chat/handler.py
View file

@ -70,7 +70,12 @@ from litellm.types.utils import (
)
from ...base import BaseLLM
from ..common_utils import AnthropicError, process_anthropic_headers, refusal_provider_specific_fields
from ..common_utils import (
AnthropicError,
process_anthropic_headers,
refusal_params,
stop_reason_provider_specific_fields,
)
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig, JsonModeRequest
if TYPE_CHECKING:
@ -926,6 +931,17 @@ class ModelResponseIterator:
def _web_search_call_snapshot(self) -> dict[str, object]:
return dict(self._web_search_calls)
def _refresh_pending_web_search_call(self, tool_use_id: str) -> bool:
if tool_use_id not in self._web_search_calls:
return False
self._web_search_calls[tool_use_id] = build_web_search_call(
tool_use_id,
self._server_tool_inputs.get(tool_use_id, {}),
{"content": []},
status="in_progress",
)
return True
def _complete_web_search_call(self, result: dict[str, object]) -> None:
tool_use_id: Final = result.get("tool_use_id")
if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
@ -1128,6 +1144,8 @@ class ModelResponseIterator:
self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
except (json.JSONDecodeError, TypeError):
pass
if self._refresh_pending_web_search_call(self._current_server_tool_id):
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
self._current_server_tool_id = None
if self.is_response_format_tool:
text = self._flush_response_format_buffer()
@ -1140,8 +1158,8 @@ class ModelResponseIterator:
# These are automatically handled by Anthropic API, we just pass them through
pass
elif type_chunk == "message_delta":
finish_reason, usage, container, refusal_fields = self._handle_message_delta(chunk)
provider_specific_fields.update(refusal_fields)
finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk)
provider_specific_fields.update(stop_reason_fields)
text = self._release_answer_close() + self._drain_split_answer_blocks()
if container:
provider_specific_fields["container"] = container
@ -1190,6 +1208,7 @@ class ModelResponseIterator:
)
text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use)
refusal_fields: Final = refusal_params(provider_specific_fields)
returned_chunk: Final = ModelResponseStream(
choices=[
@ -1201,6 +1220,7 @@ class ModelResponseIterator:
provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
thinking_blocks=(thinking_blocks if thinking_blocks else None),
reasoning_content=reasoning_content,
**refusal_fields,
),
finish_reason=finish_reason,
)
@ -1371,7 +1391,7 @@ class ModelResponseIterator:
self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) if "usage" in message_delta else None
)
container: Final = message_delta["delta"].get("container")
return finish_reason, usage, container, refusal_provider_specific_fields(message_delta["delta"])
return finish_reason, usage, container, stop_reason_provider_specific_fields(message_delta["delta"])
def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
"""

105
litellm/llms/anthropic/chat/transformation.py
View file

@ -120,13 +120,14 @@ from litellm.utils import (
)
from ..common_utils import (
REFUSAL_STOP_FIELDS,
STOP_REASON_FIELDS,
AnthropicError,
AnthropicModelInfo,
eager_input_streaming_flag,
process_anthropic_headers,
refusal_provider_specific_fields,
refusal_params,
requires_native_compaction_beta,
stop_reason_provider_specific_fields,
strip_advisor_blocks_from_messages,
tools_marked_for_a_tool_config_point,
)
@ -167,6 +168,9 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
@ -2254,6 +2258,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers,
)
@staticmethod
def _messages_within_cache_control_cap(
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
) -> Sequence[AllMessageValues]:
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return messages_within_cache_control_cap(
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
)
def transform_request(
self,
model: str,
@ -2271,9 +2285,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_messages_pt,
)
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
capped_messages: Final = (
list(
self._messages_within_cache_control_cap(
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
)
)
if drop_params
else messages
)
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
# Drop thinking param if thinking is enabled but thinking_blocks are missing
@ -2285,9 +2308,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Related issue: https://github.com/BerriAI/litellm/issues/18926
if (
optional_params.get("thinking") is not None
and messages is not None
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
and not any_assistant_message_has_thinking_blocks(messages)
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
and not any_assistant_message_has_thinking_blocks(capped_messages)
):
if litellm.modify_params:
optional_params.pop("thinking", None)
@ -2332,15 +2354,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
request_messages: Final = (
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
if _name_forward_map
else capped_messages
)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Only the leading system run becomes the top-level system prompt. A later
# system message stays in the conversation: hoisting it rewrites the cached
# prefix and re-bills the whole history at cache-write pricing (#36559).
leading_system_run, later_messages = split_leading_system_run(messages)
leading_system_run, later_messages = split_leading_system_run(request_messages)
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
# Handling anthropic API Prompt Caching
if len(anthropic_system_message_list) > 0:
@ -2362,7 +2387,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
except Exception as e:
raise AnthropicError(
status_code=400,
message=f"{e}\nReceived Messages={messages}",
message=f"{e}\nReceived Messages={request_messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(
@ -2897,28 +2922,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
web_search_results: Sequence[object],
completion_response: Mapping[str, object],
) -> list[object]:
content: Final = completion_response.get("content")
blocks: Final = content if isinstance(content, Sequence) else ()
inputs: Final = {
call_id: tool_input
for block in blocks
if isinstance(block, Mapping)
and block.get("type") == "server_tool_use"
and block.get("name") == "web_search"
and isinstance((call_id := block.get("id")), str)
and isinstance((tool_input := block.get("input")), Mapping)
blocks: Final = _RESPONSE_BLOCKS.validate_python(completion_response.get("content") or ())
results_by_id: Final = {
tool_use_id: result
for result in _RESPONSE_BLOCKS.validate_python(web_search_results)
if result.get("type") == "web_search_tool_result"
and isinstance((tool_use_id := result.get("tool_use_id")), str)
}
return [
build_web_search_call(
tool_id=tool_use_id,
tool_input=inputs.get(tool_use_id, {}),
result=result,
tool_id=call_id,
tool_input=block.get("input"),
result=results_by_id.get(call_id, _PENDING_WEB_SEARCH_RESULT),
status=None if call_id in results_by_id else "in_progress",
)
for result in web_search_results
if isinstance(result, dict)
and result.get("type") == "web_search_tool_result"
and isinstance((tool_use_id := result.get("tool_use_id")), str)
and tool_use_id in inputs
for block in blocks
if block.get("type") == "server_tool_use"
and block.get("name") == "web_search"
and isinstance((call_id := block.get("id")), str)
]
def _build_provider_specific_fields(
@ -2942,10 +2963,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if web_search_results is not None:
provider_specific_fields["web_search_results"] = web_search_results
provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
web_search_results,
completion_response,
)
web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
if web_search_results is not None or web_search_calls:
provider_specific_fields["web_search_calls"] = web_search_calls
if tool_results is not None:
provider_specific_fields["tool_results"] = tool_results
@ -2967,10 +2988,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if compaction_blocks is not None:
provider_specific_fields["compaction_blocks"] = compaction_blocks
refusal_fields: Final = refusal_provider_specific_fields(
REFUSAL_STOP_FIELDS.validate_python(completion_response)
stop_reason_fields: Final = stop_reason_provider_specific_fields(
STOP_REASON_FIELDS.validate_python(completion_response)
)
return {**provider_specific_fields, **refusal_fields}
return {**provider_specific_fields, **stop_reason_fields}
def transform_parsed_response(
self,
@ -3045,22 +3066,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if json_extra_content:
merged_text = merged_text + json_extra_content if merged_text else json_extra_content
refusal_fields: Final = refusal_params(STOP_REASON_FIELDS.validate_python(completion_response))
_message = litellm.Message(
tool_calls=tool_calls_for_message,
content=merged_text or None,
provider_specific_fields=provider_specific_fields,
thinking_blocks=thinking_blocks,
reasoning_content=reasoning_content,
**refusal_fields,
)
_message.provider_specific_fields = provider_specific_fields
if json_mode_message is not None:
refusal_fields: Final = refusal_provider_specific_fields(
REFUSAL_STOP_FIELDS.validate_python(completion_response)
stop_reason_fields: Final = stop_reason_provider_specific_fields(
STOP_REASON_FIELDS.validate_python(completion_response)
)
completion_response["stop_reason"] = "stop"
if refusal_fields:
json_mode_message.provider_specific_fields = dict(refusal_fields)
if stop_reason_fields:
json_mode_message.provider_specific_fields = dict(stop_reason_fields)
if "refusal" in refusal_fields:
json_mode_message.refusal = refusal_fields["refusal"]
if thinking_blocks:
json_mode_message.thinking_blocks = thinking_blocks
json_mode_message.reasoning_content = reasoning_content

69
litellm/llms/anthropic/common_utils.py
View file

@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
import copy
import json
import os
import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.constants import (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
ANTHROPIC_MAX_USER_ID_LENGTH,
DEFAULT_MODEL_CREATED_AT_TIME,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
def anthropic_pass_through_api_base() -> str:
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
try:
other_url: Final = httpx.URL(other)
except httpx.InvalidURL:
return False
return (url.host, url.port) == (other_url.host, other_url.port)
def anthropic_upstream_user_id_limit(url: str) -> int | None:
upstream: Final = httpx.URL(url)
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
return None
if upstream.host == "api.anthropic.com":
return ANTHROPIC_MAX_USER_ID_LENGTH
return (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
else None
)
def _has_email_shape(token: str) -> bool:
at_sign: Final = token.find("@", 1)
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
def is_anthropic_messages_url(url: str) -> bool:
"""Check whether a URL addresses Anthropic's Messages API."""
parsed_url: Final = urlparse(url)
@ -2481,7 +2518,7 @@ def create_anthropic_model_list_response(
}
class RefusalStopFields(TypedDict, total=False):
class StopReasonFields(TypedDict, total=False):
stop_reason: ReadOnly[object]
stop_details: ReadOnly[object]
@ -2498,16 +2535,38 @@ def _dict_or_empty(value: object) -> object:
return value if _is_dict(value) else _no_stop_fields()
REFUSAL_STOP_FIELDS: Final[TypeAdapter[RefusalStopFields]] = TypeAdapter(
Annotated[RefusalStopFields, BeforeValidator(_dict_or_empty)]
STOP_REASON_FIELDS: Final[TypeAdapter[StopReasonFields]] = TypeAdapter(
Annotated[StopReasonFields, BeforeValidator(_dict_or_empty)]
)
_STOP_REASONS_KEPT_RAW: Final = frozenset({"refusal", "pause_turn"})
def refusal_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
def stop_reason_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
stop_reason: Final = message.get("stop_reason")
stop_details: Final = message.get("stop_details")
if stop_details is not None:
return {"stop_reason": stop_reason, "stop_details": stop_details}
if stop_reason == "refusal":
if stop_reason in _STOP_REASONS_KEPT_RAW:
return {"stop_reason": stop_reason}
return {}
class _RefusalStopDetails(TypedDict, total=False):
explanation: ReadOnly[object]
_REFUSAL_STOP_DETAILS: Final[TypeAdapter[_RefusalStopDetails]] = TypeAdapter(
Annotated[_RefusalStopDetails, BeforeValidator(_dict_or_empty)]
)
class RefusalParams(TypedDict, total=False):
refusal: ReadOnly[str]
def refusal_params(message: Mapping[str, object]) -> RefusalParams:
if message.get("stop_reason") != "refusal":
return {}
explanation: Final = _REFUSAL_STOP_DETAILS.validate_python(message.get("stop_details")).get("explanation")
return {"refusal": explanation} if isinstance(explanation, str) and explanation else {}

52
litellm/llms/anthropic/messages/adapters/streaming_iterator.py
View file

@ -33,6 +33,7 @@ from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
AnthropicStopDetails,
AppliedEdit,
CompactionBlock,
ContentBlockDelta,
@ -284,9 +285,7 @@ class _CombinedChunkSplitter:
@staticmethod
def _is_combined(chunk: "ModelResponseStream") -> bool:
"""True if ``chunk`` carries response content AND a finish_reason."""
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
choices: Final = _optional_attr_sequence(chunk, "choices")
if not choices:
@ -302,7 +301,7 @@ class _CombinedChunkSplitter:
or _optional_attr(delta, "tool_calls")
or _optional_attr(delta, "reasoning_content")
or _optional_attr(delta, "thinking_blocks")
or openai_chat_refusal_text(delta)
or refusal_reply_text(delta)
)
_PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
@ -659,6 +658,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
self.compaction_block = compaction_block
self.iterations_usage = iterations_usage
self._refusal_text: str = ""
self._paused_turn: bool = False
self._claude_refusal_stop_details: AnthropicStopDetails | None = None
self.sent_compaction_block: bool = False
# Per-phase flags so the compaction block's start/delta/stop events
# are emitted (and the public state machine is advanced) in
@ -953,7 +954,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
current_content_block_index=self.current_content_block_index,
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
)
processed_chunk = self._with_refusal_stop_details(processed_chunk)
processed_chunk = self._with_stop_reason_fields(processed_chunk)
# Check if this is a usage chunk and we have a held stop_reason chunk
if will_merge_into_held:
@ -1188,7 +1189,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
current_content_block_index=self.current_content_block_index,
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
)
processed_chunk = self._with_refusal_stop_details(processed_chunk)
processed_chunk = self._with_stop_reason_fields(processed_chunk)
# Check if this is a usage chunk and we have a held stop_reason chunk
if will_merge_into_held:
@ -1406,14 +1407,22 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
def _increment_content_block_index(self):
self.current_content_block_index += 1
def _with_refusal_stop_details(
def _with_stop_reason_fields(
self,
processed_chunk: ContentBlockDelta | MessageBlockDelta,
) -> ContentBlockDelta | MessageBlockDelta:
if processed_chunk["type"] != "message_delta" or not self._refusal_text:
if processed_chunk["type"] != "message_delta":
return processed_chunk
delta: Final = processed_chunk["delta"]
if delta.get("stop_reason") == "max_tokens":
if self._paused_turn:
paused_delta: Final[MessageDelta] = {**delta, "stop_reason": "pause_turn"}
paused_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": paused_delta}
return paused_chunk
if delta.get("stop_reason") == "refusal" and self._claude_refusal_stop_details is not None:
claude_refusal_delta: Final[MessageDelta] = {**delta, "stop_details": self._claude_refusal_stop_details}
claude_refusal_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": claude_refusal_delta}
return claude_refusal_chunk
if not self._refusal_text or delta.get("stop_reason") == "max_tokens":
return processed_chunk
from litellm.llms.anthropic.messages.native.utils import (
refusal_stop_details,
@ -1470,9 +1479,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
@staticmethod
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
choice: Final = chunk.choices[0]
if choice.finish_reason is not None:
@ -1482,7 +1489,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
return False
if getattr(delta, "content", None):
return False
if openai_chat_refusal_text(delta):
if refusal_reply_text(delta):
return False
if getattr(delta, "reasoning_content", None):
return False
@ -1497,6 +1504,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
return False
return True
def _record_anthropic_stop_fields(self, delta: object) -> None:
from litellm.llms.anthropic.messages.native.utils import (
anthropic_refusal_stop_details,
anthropic_stop_reason,
)
if anthropic_stop_reason(delta) == "pause_turn":
self._paused_turn = True
claude_refusal_stop_details: Final = anthropic_refusal_stop_details(delta)
if claude_refusal_stop_details is not None:
self._claude_refusal_stop_details = claude_refusal_stop_details
def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:
"""
Determine if we should start a new content block based on the processed chunk.
@ -1507,16 +1526,15 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
- Different content types in the response
- Specific markers in the content
"""
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
from .transformation import LiteLLMAnthropicMessagesAdapter
self._record_anthropic_stop_fields(chunk.choices[0].delta)
if chunk.choices[0].finish_reason is not None:
return False
refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
refusal_text: Final = refusal_reply_text(chunk.choices[0].delta)
if refusal_text is not None:
self._refusal_text = self._refusal_text + refusal_text

41
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -150,7 +150,9 @@ from litellm.llms.anthropic.messages.native.mid_conversation_system import (
is_system_role_message,
)
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
anthropic_refusal_stop_details,
anthropic_stop_reason,
refusal_reply_text,
refusal_stop_details,
)
from litellm.types.llms.anthropic import (
@ -398,6 +400,10 @@ class LiteLLMAnthropicMessagesAdapter:
return None
def _streamed_tool_call_signature(self, tool_call_id: str, tool_call: object) -> str | None:
id_parts: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
return id_parts[1] if len(id_parts) > 1 else self._extract_signature_from_tool_call(tool_call)
def _extract_signature_from_tool_use_content(self, content: Mapping[str, object]) -> str | None:
"""
Extract signature from a tool_use content block's provider_specific_fields.
@ -1467,7 +1473,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_content.append(
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
)
if (refusal_text := openai_chat_refusal_text(choice.message)) is not None:
if (refusal_text := refusal_reply_text(choice.message)) is not None:
new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump())
# Handle tool calls (in parallel to text content)
if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0:
@ -1634,7 +1640,15 @@ class LiteLLMAnthropicMessagesAdapter:
tool_name_mapping=tool_name_mapping,
)
refusal_text: Final = next(
(text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None),
(text for choice in response.choices if (text := refusal_reply_text(choice.message)) is not None),
None,
)
claude_refusal_stop_details: Final = next(
(
details
for choice in response.choices
if (details := anthropic_refusal_stop_details(choice.message)) is not None
),
None,
)
@ -1643,8 +1657,10 @@ class LiteLLMAnthropicMessagesAdapter:
## extract finish reason
openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop"
translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
openai_finish_reason=openai_finish_reason
translated_finish_reason: Final = (
"pause_turn"
if response.choices and anthropic_stop_reason(response.choices[0].message) == "pause_turn"
else self._translate_openai_finish_reason_to_anthropic(openai_finish_reason=openai_finish_reason)
)
anthropic_finish_reason: Final = (
"compaction"
@ -1686,7 +1702,11 @@ class LiteLLMAnthropicMessagesAdapter:
usage=anthropic_usage,
content=anthropic_content,
stop_reason=anthropic_finish_reason,
stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None),
stop_details=(
(claude_refusal_stop_details or refusal_stop_details(refusal_text))
if anthropic_finish_reason == "refusal"
else None
),
)
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
@ -1712,10 +1732,7 @@ class LiteLLMAnthropicMessagesAdapter:
):
raw_id = choice.delta.tool_calls[0].id or str(uuid.uuid4())
tool_name = choice.delta.tool_calls[0].function.name or ""
thought_sig: str | None = None
if THOUGHT_SIGNATURE_SEPARATOR in raw_id:
parts = raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
thought_sig = parts[1] if len(parts) > 1 else None
thought_sig = self._streamed_tool_call_signature(raw_id, choice.delta.tool_calls[0])
tool_block: dict[str, object] = {
"type": "tool_use",
"id": normalize_anthropic_tool_use_id(raw_id),
@ -1727,7 +1744,7 @@ class LiteLLMAnthropicMessagesAdapter:
"signature": thought_sig,
}
return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text(
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or refusal_reply_text(
choice.delta
) is not None:
return "text", TextBlock(type="text", text="")
@ -1810,7 +1827,7 @@ class LiteLLMAnthropicMessagesAdapter:
return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
else:
refusal_text: Final = "".join(
refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None
refusal for choice in choices if (refusal := refusal_reply_text(choice.delta)) is not None
)
return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text)

14
litellm/llms/anthropic/messages/native/handler.py
View file

@ -19,6 +19,13 @@ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheCont
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.reasoning_effort_capability import (
RequestedReasoningEffort,
deployment_reasoning_effort_levels,
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
)
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
strip_provider_specific_fields_from_anthropic_messages,
@ -31,13 +38,6 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
from litellm.router_utils.reasoning_effort_capability import (
RequestedReasoningEffort,
deployment_reasoning_effort_levels,
keeps_effort,
model_group_from_metadata,
resolve_deployment_reasoning_effort,
)
from litellm.types.llms.anthropic import AllAnthropicToolsValues
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
from litellm.types.llms.anthropic_messages.anthropic_response import (

2
litellm/llms/anthropic/messages/native/interceptors/advisor.py
View file

@ -22,7 +22,7 @@ import litellm
import litellm.constants as _c
from litellm.litellm_core_utils.url_utils import validate_url
from litellm.llms.anthropic.common_utils import strip_advisor_blocks_from_messages
from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure
from litellm.router_api import mark_advisor_orchestration_failure
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,

30
litellm/llms/anthropic/messages/native/utils.py
View file

@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Final, cast, get_type_hints
from pydantic import JsonValue, TypeAdapter, ValidationError
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, with_config
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.core_helpers import as_str_mapping
@ -162,6 +162,34 @@ def openai_chat_refusal_text(message_or_delta: object) -> str | None:
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal")
def anthropic_stop_reason(message_or_delta: object) -> str | None:
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "stop_reason")
def refusal_reply_text(message_or_delta: object) -> str | None:
if anthropic_stop_reason(message_or_delta) == "refusal":
return None
return openai_chat_refusal_text(message_or_delta)
@with_config(ConfigDict(extra="allow"))
class _AnthropicRefusalStopDetails(AnthropicStopDetails, total=False):
pass
_ANTHROPIC_REFUSAL_STOP_DETAILS: Final = TypeAdapter(_AnthropicRefusalStopDetails)
def anthropic_refusal_stop_details(message_or_delta: object) -> AnthropicStopDetails | None:
provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None)
if _mapping_str_field(provider_specific_fields, "stop_reason") != "refusal":
return None
try:
return _ANTHROPIC_REFUSAL_STOP_DETAILS.validate_python(_mapping_field(provider_specific_fields, "stop_details"))
except ValidationError:
return refusal_stop_details(openai_chat_refusal_text(message_or_delta))
def _responses_message_refusal_text(item: object) -> str | None:
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal

272
litellm/llms/anthropic/messages/stream_assembly.py
View file

@ -155,6 +155,8 @@ def collapse_pure_text_chunks(
continue
json_str = ev[idx + len("data:") :].strip()
if not json_str.startswith("{"):
continue
try:
data = json.loads(json_str)
except (json.JSONDecodeError, ValueError):
@ -206,6 +208,11 @@ def collapse_pure_text_chunks(
return out
def _has_non_object_data(event_str: str) -> bool:
data_start: Final = event_str.find("data:")
return data_start != -1 and not event_str[data_start + len("data:") :].lstrip().startswith("{")
def build_complete_streaming_response_legacy(
all_chunks: Sequence[str | bytes],
litellm_logging_obj: LiteLLMLoggingObj,
@ -243,6 +250,8 @@ def build_complete_streaming_response_legacy(
# text payload happens to contain "[DONE]" is not dropped.
if any(line.strip() == "data: [DONE]" for line in event_str.split("\n")):
continue
if _has_non_object_data(event_str):
continue
transformed_openai_chunk = anthropic_model_response_iterator.convert_str_chunk_to_generic_chunk(
chunk=event_str
)
@ -268,21 +277,25 @@ def build_complete_streaming_response_legacy(
return complete_streaming_response
def extract_sse_data(event_str: str) -> dict | None:
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
def _sse_data_payload(event_str: str) -> str | None:
for line in split_sse_lines(event_str):
stripped = line.strip()
if stripped.startswith("data:"):
payload = stripped[len("data:") :].strip()
if not payload or payload == "[DONE]":
return None
try:
return json.loads(payload)
except (ValueError, TypeError):
return None
return stripped[len("data:") :].strip()
return None
def extract_sse_data(event_str: str) -> dict | None:
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
payload: Final = _sse_data_payload(event_str)
if payload is None or not payload.startswith("{"):
return None
try:
return json.loads(payload)
except (ValueError, TypeError):
return None
def build_usage_only_response_from_chunks(
all_chunks: Sequence[str | bytes],
model: str,
@ -415,27 +428,81 @@ _ANTHROPIC_EVENT_TYPES: Final = frozenset(
)
_ERROR_PAYLOAD_KEYS: Final = frozenset({"type", "error", "request_id"})
def _is_anthropic_event_type(event_type: object) -> bool:
return isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES
def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool:
return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks)
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
raw: Final = b"".join(
def _joined_sse_bytes(all_chunks: Sequence[object]) -> bytes:
return b"".join(
chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
for chunk in all_chunks
if isinstance(chunk, (str, bytes))
)
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
try:
return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False)
return codecs.getincrementaldecoder("utf-8")().decode(_joined_sse_bytes(all_chunks), final=False)
except UnicodeDecodeError:
return None
def _sse_event_data(event: str) -> Mapping[str, object] | None:
return as_str_mapping(extract_sse_data(event))
_JSON_ESCAPED_LINE_BREAKS: Final = MappingProxyType(
str.maketrans({"\x85": "\\u0085", "\u2028": "\\u2028", "\u2029": "\\u2029"})
)
def _sse_frames(sse_stream: str) -> Sequence[str]:
return "\n".join(split_sse_lines(sse_stream)).split("\n\n")
def _frames_of_the_widest_reader(sse_stream: str) -> Sequence[str]:
"""The stream's frames as httpx's ``str.splitlines`` reads them, past a form feed or U+2028 in any line."""
return "\n".join(sse_stream.splitlines()).split("\n\n")
def _sse_field(line: str) -> tuple[str, str]:
name, _, value = line.partition(":")
return name, value.removeprefix(" ")
def _sse_fields(frame: str) -> tuple[tuple[str, str], ...]:
"""The frame's fields as a client reads them: a name matches exactly and a comment is no field."""
return tuple(_sse_field(line) for line in frame.split("\n") if line and not line.startswith(":"))
def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
match fields:
case () | (("event", _),) | (("data", "[DONE]"),):
return True
case _:
return False
def _json_mapping(payload: str) -> Mapping[str, object] | None:
try:
value: Final[object] = json.loads(payload)
except ValueError:
return None
return as_str_mapping(value)
def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]:
return tuple(
event_data
for event in split_sse_chunk_into_events(sse_stream)
if (event_data := extract_sse_data(event)) is not None
if (event_data := _sse_event_data(event)) is not None
)
@ -444,7 +511,8 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
(
message
for event_data in _parsed_sse_events(sse_stream)
if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict)
if event_data.get("type") == "message_start"
and (message := as_str_mapping(event_data.get("message"))) is not None
),
None,
)
@ -482,11 +550,151 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return False
return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream))
return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream))
def _is_json_no_reader_can_take(payload: str) -> bool:
try:
value: Final[object] = json.loads(payload)
except ValueError:
return False
if value is None:
return False
event: Final = as_str_mapping(value)
return event is None or isinstance(event.get("type"), (list, dict))
def has_unreadable_json_event(all_chunks: Sequence[object]) -> bool:
"""Whether a frame's data is JSON that no reader of these streams can take as an event."""
sse_stream: Final = codecs.getincrementaldecoder("utf-8")(errors="replace").decode(
_joined_sse_bytes(all_chunks), final=False
)
return any(_is_json_no_reader_can_take(_sse_data_payload(event) or "") for event in _sse_frames(sse_stream))
def anthropic_sse_event_type(event: str) -> object:
return next((event_data.get("type") for event_data in _parsed_sse_events(event)), None)
def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]:
"""The string ``type`` of every event the stream sent."""
event_types: Final = (event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
return frozenset(event_type for event_type in event_types if isinstance(event_type, str))
def _is_named_error_event(event: str) -> bool:
names: Final = tuple(value for name, value in _sse_fields(event) if name == "event")
return names[-1:] == ("error",)
def _is_error_event(event: str) -> bool:
event_type: Final = anthropic_sse_event_type(event)
return event_type == "error" or (_is_named_error_event(event) and not _is_anthropic_event_type(event_type))
def _frames_a_client_dispatches(sse_stream: str) -> Sequence[str]:
return "\n".join((*split_sse_lines(sse_stream), "")).split("\n\n")[:-1]
def _error_events(sse_stream: str) -> tuple[str, ...]:
return tuple(event for event in _frames_a_client_dispatches(sse_stream) if _is_error_event(event))
def _plain_error_event(event: str) -> str:
payload: Final = _sse_data_payload(event) or ""
one_line: Final = payload.translate(_JSON_ESCAPED_LINE_BREAKS) if _json_mapping(payload) is not None else payload
return "\n".join(("event: error", *(f"data: {line}" for line in one_line.splitlines()[:1])))
def anthropic_sse_error_events(all_chunks: Sequence[object]) -> tuple[str, ...]:
"""The ``error`` events of the stream, each as an ``event:`` line and the ``data:`` line assembly read.
A stream whose upstream fails after it started ends with an ``error`` event in place of
``message_stop``. That frame is the outcome the client is owed, not content, so assembly through
the upstream error leaves it out and a caller re-emitting a rewritten response puts it back,
without any other line of the frame, since that is content no guardrail read.
A frame counts by its data ``type``, or by its SSE ``event:`` name unless its data is another
Anthropic event, since a client SDK raises on the name alone and an upstream that is not
Anthropic may send anything as the data.
"""
return tuple(_plain_error_event(event) for event in _error_events(_joined_sse_stream(all_chunks) or ""))
def anthropic_sse_stopped_block_index(event: str) -> object:
return next(
(
event_data.get("index")
for event_data in _parsed_sse_events(event)
if event_data.get("type") == "content_block_stop"
),
None,
)
def _block_index(event_data: Mapping[str, object]) -> int | None:
index: Final = event_data.get("index")
return index if isinstance(index, int) else None
def _started_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
starts: Final = (event for event in events if event.get("type") == "content_block_start")
return MappingProxyType(
{
index: block
for event in starts
if (index := _block_index(event)) is not None
and (block := as_str_mapping(event.get("content_block"))) is not None
}
)
def anthropic_sse_started_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
"""The ``content_block`` each ``content_block_start`` of the stream opened, by block index."""
return _started_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
def _unfinished_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
stopped: Final = frozenset(_block_index(event) for event in events if event.get("type") == "content_block_stop")
return MappingProxyType({index: block for index, block in _started_blocks(events).items() if index not in stopped})
def anthropic_sse_unfinished_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
"""The blocks the stream opened and never stopped, which only a failed upstream leaves."""
return _unfinished_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
def _sse_events_with_data(sse_stream: str) -> tuple[tuple[str, Mapping[str, object]], ...]:
return tuple(
(event, event_data)
for event in split_sse_chunk_into_events(sse_stream)
if (event_data := _sse_event_data(event)) is not None
)
def _content_events_before_upstream_error(sse_stream: str) -> tuple[str, ...]:
"""The stream's events minus its upstream ``error`` and every cut-off block assembly cannot read.
A cut-off text block is still text the client was sent, so it stays. A cut-off tool call's
input is a JSON prefix, and a cut-off thinking block has no signature, so neither is a block the
client can act on, and both are left out.
"""
if not _error_events(sse_stream):
return (sse_stream,)
events: Final = _sse_events_with_data(sse_stream)
cut_off: Final = frozenset(
index
for index, block in _unfinished_blocks(tuple(event_data for _, event_data in events)).items()
if block.get("type") != "text"
)
return tuple(
event
for event, event_data in events
if event_data.get("type") != "error" and _block_index(event_data) not in cut_off
)
def assemble_anthropic_sse_stream(
all_chunks: Sequence[object], *, restore_identity: bool = False
all_chunks: Sequence[object], *, restore_identity: bool = False, through_upstream_error: bool = False
) -> ModelResponse | None:
"""Assemble raw Anthropic SSE frames into a ModelResponse.
@ -494,6 +702,11 @@ def assemble_anthropic_sse_stream(
assembler does not carry through. It is off by default so callers that re-emit the assembled
response keep the wire shape they had before this helper was shared. The writes land on a
freshly built object that is unreachable from caller state until returned.
A stream whose upstream failed after it started ends with an ``error`` event in place of
``message_stop`` and assembles to None, unless ``through_upstream_error`` asks for the content
the client was sent before the failure. That is for a guardrail that scans the stream and then
passes the upstream error on; a caller recording what was served keeps None.
"""
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
@ -502,9 +715,12 @@ def assemble_anthropic_sse_stream(
if message_start is None:
return None
model: Final = message_start.get("model") if restore_identity else None
content_events: Final = (
_content_events_before_upstream_error(sse_stream) if through_upstream_error else (sse_stream,)
)
try:
assembled: Final = build_complete_streaming_response(
all_chunks=(sse_stream,),
all_chunks=content_events,
litellm_logging_obj=None, # pyright: ignore[reportArgumentType] # only forwarded to stream_chunk_builder, which accepts None
model=model if isinstance(model, str) else "",
)
@ -522,6 +738,21 @@ def assemble_anthropic_sse_stream(
return assembled
def _is_error_payload(payload: str) -> bool:
event: Final = _json_mapping(payload)
if event is None or not _ERROR_PAYLOAD_KEYS.issuperset(event):
return False
return event.get("type") == "error" or ("type" not in event and isinstance(event.get("error"), Mapping))
def _is_error_frame(fields: Sequence[tuple[str, str]]) -> bool:
match fields:
case (("data", payload),) | (("event", "error"), ("data", payload)):
return _is_error_payload(payload)
case _:
return False
def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
"""Whether the buffered stream carries nothing but error frames.
@ -537,7 +768,6 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return False
events: Final = _parsed_sse_events(sse_stream)
return len(events) > 0 and all(
event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events
)
frames: Final = tuple(_sse_fields(frame) for frame in _frames_of_the_widest_reader(sse_stream))
with_data: Final = tuple(fields for fields in frames if not _carries_no_data(fields))
return len(with_data) > 0 and all(_is_error_frame(fields) for fields in with_data)

2
litellm/llms/anthropic/messages/utils.py
View file

@ -214,7 +214,7 @@ def normalize_reasoning_effort_value(
if chain is None:
return effort
from litellm.router_utils.reasoning_effort_capability import (
from litellm.litellm_core_utils.reasoning_effort_capability import (
deployment_rounds_reasoning_effort,
resolve_supported_reasoning_efforts,
)

34
litellm/llms/base_llm/decisions/transformation.py
View file

@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
assert_never(question)
def ir_to_systemone_request(
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
if request.audio and not supports_audio:
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
if request.videos and not supports_video:
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
state: Final = _systemone_state(request.input)
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
images: Final = input_images(request.input)
@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
)
def catalog_mode(model: str, custom_llm_provider: str) -> str | None:
try:
key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"]
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list
return None
entry: Final = GetModelCostMap.loaded_model_cost_map().get(key)
mode: Final = entry.get("mode") if entry is not None else None
return mode if isinstance(mode, str) else None
def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
try:
model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC):
model=model, custom_llm_provider=custom_llm_provider
)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
if deployment_flag is not None:
return deployment_flag
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
model=model, custom_llm_provider=custom_llm_provider
)
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
if deployment_flag is not None:
return deployment_flag
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
model=model, custom_llm_provider=custom_llm_provider
)
@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC):
request: DecisionsIRRequest,
custom_llm_provider: str,
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
return ir_to_systemone_request(
self.request_model(model),
request,
supports_audio=self.supports_audio_input(model, custom_llm_provider),
supports_video=self.supports_video_input(model, custom_llm_provider),
)
return ir_to_systemone_request(self.request_model(model), request)
def unwrap_response(self, payload: object) -> object:
return payload

35
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -25,6 +25,7 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
filter_exceptions_from_params,
filter_internal_params,
map_finish_reason,
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
return data
@staticmethod
def _claude_messages_within_cache_control_cap(
messages: Sequence[AllMessageValues],
capability_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
return messages
from litellm.integrations.anthropic_cache_control_hook import (
messages_within_cache_control_cap,
tool_message_with_single_cache_mark,
)
return messages_within_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
)
async def _async_transform_request(
self,
model: str,
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present

2
litellm/llms/clf_ai_gateway/chat/transformation.py
View file

@ -4,8 +4,8 @@ import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.secret_managers.main import get_secret_str
SUPPORTED_OPENAI_PARAMS: Final = (

8
litellm/llms/compaction.py
View file

@ -9,7 +9,7 @@ from pydantic import TypeAdapter
from litellm.types.llms.openai import AllMessageValues
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
CompactionProtocol: TypeAlias = Literal["chat", "messages"]
_MAPPING: Final = TypeAdapter(Mapping[str, object])
@ -32,7 +32,9 @@ def get_native_compaction_provider(params: Mapping[str, object]) -> NativeCompac
return compaction if compaction.supports_native_compaction(params) else None
async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mapping[str, object]) -> Mapping[str, object]:
async def dispatch(
router: RoutingContext, protocol: CompactionProtocol, payload: Mapping[str, object]
) -> Mapping[str, object]:
if protocol == "messages":
return _MAPPING.validate_python(
await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload)
@ -45,4 +47,4 @@ async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mappin
{key: value for key, value in payload.items() if key not in ("model", "messages", "stream")}
),
)
return _MAPPING.validate_python(response.model_dump())
return _MAPPING.validate_python(response.model_dump()) # pyright: ignore[reportAttributeAccessIssue] # stream=False: acompletion returns a ModelResponse

15
litellm/llms/custom_httpx/aiohttp_transport.py
View file

@ -110,6 +110,15 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]:
raise mapped_exc(message) from exc
def _unread_payload(content: object) -> bytes:
"""Bytes the upstream sent before the connection dropped that aiohttp buffered but never handed out, since its
reads raise the stored payload error ahead of the buffer."""
if not isinstance(content, aiohttp.StreamReader):
return b""
buffered: Final = b"".join(content._buffer) # pyright: ignore[reportPrivateUsage] # no public read drains it
return buffered[content._buffer_offset :] # pyright: ignore[reportPrivateUsage] # see above
def _release_transport_anchor(_transport: object) -> None:
"""Release a transport anchored to a response stream. See ``_anchor_transport_to``."""
@ -144,6 +153,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
raise httpx.ReadError(str(e)) from e
except aiohttp.http_exceptions.TransferEncodingError as e:
raise httpx.ReadError(str(e)) from e
except aiohttp.ClientPayloadError:
unread: Final = _unread_payload(self._aiohttp_response.content)
if unread:
yield unread
with map_aiohttp_exceptions():
raise
except Exception:
# For other exceptions, use the normal mapping
with map_aiohttp_exceptions():

39
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -131,6 +131,8 @@ from litellm.responses.streaming_iterator import (
ResponsesAPIStreamingIterator,
ResponsesWebSocketStreaming,
SyncResponsesAPIStreamingIterator,
TokenLimitCallback,
with_token_limit_checks,
)
from litellm.types.containers.main import (
ContainerFileListResponse,
@ -598,10 +600,10 @@ def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
)
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
"""Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
enforcement, so the Responses WebSocket loop can charge every
``response.create`` frame, not just the connection's first one.
def _ws_callbacks_exposing(method_name: str) -> tuple[object, ...]:
"""Duck-type discover proxy hooks exposing a per-frame check, so the
Responses WebSocket loop can run it on every ``response.create`` frame,
not just the connection's first one.
Uses duck-typing on ``litellm.callbacks`` (rather than importing the
proxy hook directly) to avoid a layering violation (SDK importing from
@ -612,13 +614,28 @@ def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
callbacks: Final = cast( # cast-ok: callback registry is inspected before protocol use
Sequence[object], _litellm.callbacks
)
return tuple(callback for callback in callbacks if callable(getattr(callback, method_name, None)))
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
return tuple(
cast(ProjectQuotaCallback, callback) # cast-ok: required callback method is callable
for callback in callbacks
if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
for callback in _ws_callbacks_exposing("enforce_project_io_token_quota_for_frame")
)
def _collect_ws_token_limit_callbacks() -> tuple[TokenLimitCallback, ...]:
return tuple(
cast(TokenLimitCallback, callback) # cast-ok: required callback method is callable
for callback in _ws_callbacks_exposing("enforce_token_limits_for_frame")
)
def _ws_model_group(litellm_metadata: Mapping[str, object] | None, model: str) -> str:
model_group: Final = (litellm_metadata or {}).get("model_group")
return model_group if isinstance(model_group, str) and model_group else model
def _unwrap_provider_error(error: Exception) -> Exception:
if not isinstance(error, httpx.TransportError):
return error
@ -7285,6 +7302,12 @@ class BaseLLMHTTPHandler:
- Forwards events over the websocket connection
"""
_ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
_ws_turn_admission: Final = with_token_limit_checks(
turn_admission,
_collect_ws_token_limit_callbacks(),
user_api_key_dict,
_ws_model_group(litellm_metadata, model),
)
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
@ -7332,7 +7355,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
first_message=first_message,
quota_callbacks=_ws_quota_callbacks,
turn_admission=turn_admission,
turn_admission=_ws_turn_admission,
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
frame_guard=frame_guard,
**kwargs,
@ -7429,7 +7452,7 @@ class BaseLLMHTTPHandler:
authorized_model=model,
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
turn_admission=turn_admission,
turn_admission=_ws_turn_admission,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

2
litellm/llms/databricks/embed/handler.py
View file

@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
aembedding=None,
custom_endpoint: bool | None = None,
headers: dict | None = None,
drop_params: bool = False,
) -> EmbeddingResponse:
# Check for custom user agent in optional_params or environment
# This allows partners building on LiteLLM to set their own telemetry
@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
aembedding=aembedding,
custom_endpoint=True,
headers=headers,
drop_params=drop_params,
)

5
litellm/llms/deepinfra/chat/transformation.py
View file

@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
"""
for message in messages:
copied_messages: Final = [message.copy() for message in messages]
for message in copied_messages:
if message.get("role") == "tool":
content = message.get("content")
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
# For complex content, serialize the entire array as JSON string
message["content"] = json.dumps(content)
return messages
return copied_messages
@overload
def _transform_messages(

38
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
OpenAIChatCompletionToolParam,
)
from litellm.types.utils import (
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from ...openai.common_utils import without_cache_control
from ..common_utils import (
FIREROUTER,
FireworksAIException,
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
for tool in tools:
if tool.get("type") != "function":
continue
function = tool["function"]
function.pop("strict", None)
params = function.get("parameters")
if isinstance(params, dict):
unpack_legacy_defs(params)
return tools
return [self._fireworks_tool(tool) for tool in tools]
@staticmethod
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
if tool.get("type") != "function":
return tool
function: Final = tool["function"]
params: Final = function.get("parameters")
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
)
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
if isinstance(params, dict)
else unstrict_function
)
return {**tool, "function": new_function}
def _transform_messages_helper(
self, messages: list[AllMessageValues], model: str, litellm_params: dict
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
"""
Strip fields not permitted by FireworksAI from messages.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
for message in messages:
if message["role"] == "user":
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
model=model,
llm_provider="fireworks_ai",
)
filter_value_from_dict(cast(dict, message), "cache_control")
return [strip_litellm_response_only_fields(message) for message in messages]
return [
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
for message in messages
]
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
# this index requires a full scan of model_cost (tens of thousands of

9
litellm/llms/groq/chat/transformation.py
View file

@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
for idx, message in enumerate(messages):
transformed_messages: Final = list(messages)
for idx, message in enumerate(transformed_messages):
"""
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
"""
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
for k, v in _message.items():
if v is not None:
new_message[k] = v
messages[idx] = strip_litellm_internal_message_fields(new_message)
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
if is_async:
return super()._transform_messages(messages=messages, model=model, is_async=True)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
else:
return super()._transform_messages(messages=messages, model=model, is_async=False)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
def transform_messages(
self,

3
litellm/llms/hosted_vllm/chat/transformation.py
View file

@ -2,6 +2,7 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
import copy
import json
from collections.abc import Coroutine
from typing import Final, Literal, cast, overload
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
) -> dict:
_tools = non_default_params.pop("tools", None)
if _tools is not None:
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
_tools = remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)

11
litellm/llms/mistral/chat/transformation.py
View file

@ -25,16 +25,16 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_litellm_internal_message_fields,
strip_none_values_from_message,
)
from litellm.litellm_core_utils.reasoning_effort_capability import (
declared_reasoning_efforts_for_model,
nearest_declared_reasoning_effort,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from litellm.router_utils.reasoning_effort_capability import (
declared_reasoning_efforts_for_model,
nearest_declared_reasoning_effort,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.mistral import (
MistralCacheKeyParams,
@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
if _name is not None:
# Remove name if not a tool message
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
message.pop("name", None)
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
return message

2
litellm/llms/oci/chat/transformation.py
View file

@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.url_utils import join_api_base
@ -54,7 +55,6 @@ from litellm.llms.oci.common_utils import (
sign_oci_request,
validate_oci_environment,
)
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.types.llms.oci import (
CohereChatRequest,
OCIChatRequestPayload,

186
litellm/llms/openai/chat/gpt_transformation.py
View file

@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
extract_reasoning_content,
handle_invalid_parallel_tool_calls,
@ -64,7 +64,7 @@ from litellm.types.utils import (
)
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
if TYPE_CHECKING:
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
return value if isinstance(value, str) else None
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
return drops_unsupported_params(litellm_params.get("drop_params"))
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
def _claude_messages_within_cache_control_cap(
model: str,
messages: Sequence[AllMessageValues],
tools: Sequence["ChatCompletionToolParam"] | None,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
return messages
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
try:
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
return content_copy
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_id: Final = content_item.get("file_id")
content_copy: Final = content_item.copy()
file_id: Final = content_copy.get("file_id")
if file_id is not None: # check for file id being url done in _handle_pdf_url
base64_data: Final = await async_convert_url_to_base64(file_id)
content_item["file_data"] = base64_data
content_item["filename"] = "my_file.pdf"
content_item.pop("file_id")
return content_item
content_copy["file_data"] = base64_data
content_copy["filename"] = "my_file.pdf"
content_copy.pop("file_id")
return content_copy
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_data: Final = content_item.get("file_data")
filename: Final = content_item.get("filename")
if file_data is not None and filename is None:
content_item["filename"] = "my_file.pdf"
return {**content_item, "filename": "my_file.pdf"}
return content_item
@staticmethod
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
image_url: Final = content_item["image_url"]
if isinstance(image_url, str):
return {**content_item, "image_url": {"url": image_url}}
if not isinstance(image_url, dict):
return content_item
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
)
return {**content_item, "image_url": new_image_url_obj}
@staticmethod
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
return {**content_item, "file": new_file_obj}
def _apply_common_transform_content_item(
self,
content_item: OpenAIMessageContentListBlock,
) -> OpenAIMessageContentListBlock:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
if content_item.get("type") == "image_url":
content_item = cast(ChatCompletionImageObject, content_item)
if isinstance(content_item["image_url"], str):
content_item["image_url"] = {
"url": content_item["image_url"],
}
elif isinstance(content_item["image_url"], dict):
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
)
content_item["image_url"] = new_image_url_obj
elif content_item.get("type") == "file":
content_item = cast(ChatCompletionFileObject, content_item)
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
content_item["file"] = new_file_obj
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
if content_item.get("type") == "file":
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
return content_item
def _transform_content_item(
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
async def _async_transform():
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content_types):
message_content_types[i] = await self._async_transform_content_item(
cast(OpenAIMessageContentListBlock, content_item),
)
return hoisted_messages
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
if is_async:
return _async_transform()
else:
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content):
message_content_types[i] = self._transform_content_item(
cast(OpenAIMessageContentListBlock, content_item)
)
return hoisted_messages
return [self._user_message_for_openai(message) for message in hoisted_messages]
@staticmethod
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
message_content: Final = message.get("content")
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
return None
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
)
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
)
def transform_messages(
self,
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
messages: list[AllMessageValues],
tools: list["ChatCompletionToolParam"] | None = None,
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
from litellm.types.llms.openai import ChatCompletionToolParam
for i, message in enumerate(messages):
messages[i] = cast(
AllMessageValues,
filter_value_from_dict(message, "cache_control"),
)
if tools is not None:
for i, tool in enumerate(tools):
tools[i] = cast(
ChatCompletionToolParam,
filter_value_from_dict(tool, "cache_control"),
)
return messages, tools
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
new_tools: Final = (
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
if tools is not None
else None
)
return new_messages, new_tools
def translate_developer_role_to_system_role(
self,
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools: Sequence["ChatCompletionToolParam"] | None,
prompt_cache_options: object,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
"""
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
model, custom_llm_provider, api_base, prompt_cache_options
)
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
return messages, tools
drops_breakpoints: Final = not reads_breakpoints and (
litellm.drop_params is True or litellm_params.get("drop_params") is True
)
return (
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
tools,
)
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
renamed: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if reads_breakpoints
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
if drops_breakpoints
else list(tools)
)
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
model=model, messages=renamed, tools=kept_tools
)
return (
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
sent_tools,
)
def _sanitized_tools_update_for_openai(
self,
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools

9
litellm/llms/openai/chat/o_series_transformation.py
View file

@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
- role: system ==> translate to role 'user'
"""
_supports_system_messages: Final = supports_system_messages(model, "openai")
for i, message in enumerate(messages):
transformed_messages: Final = list(messages)
for i, message in enumerate(transformed_messages):
if message["role"] == "system" and not _supports_system_messages:
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
messages[i] = new_message # Replace the old message with the new one
transformed_messages[i] = new_message # Replace the old message with the new one
if is_async:
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
else:
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))

44
litellm/llms/openai/common_utils.py
View file

@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
"""
import asyncio
import copy
import hashlib
import inspect
import json
@ -31,7 +32,7 @@ if TYPE_CHECKING:
import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
)
def _is_plain_dict(value: object) -> bool:
return isinstance(value, dict)
def _is_list(value: object) -> bool:
return isinstance(value, list)
def _contains_cache_control(value: object) -> bool:
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
while pending:
node = pending.pop()
mapping = as_str_mapping(node)
if mapping is not None:
if "cache_control" in mapping:
return True
pending.extend(mapping.values())
elif _is_list(node):
pending.extend(iter_items(node))
return False
def without_cache_control(item: object) -> object:
"""Return `item` with every `cache_control` key removed, without touching the original.
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
lists, so the caller's own message, tool or input object has to be copied first: it
may reuse the same list on a provider that does support prompt caching. The copy is
skipped when there is nothing to strip, which keeps the object identity the
pass-through paths rely on and avoids a deep copy on every request.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
if not _is_plain_dict(item) or not _contains_cache_control(item):
return item
copied: Final = copy.deepcopy(item)
if isinstance(copied, dict):
filter_value_from_dict(copied, "cache_control")
return copied
def _get_client_init_params(cls: type) -> tuple[str, ...]:
"""Extract __init__ parameter names (excluding 'self') from a class."""
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")

4
litellm/llms/openai/decisions/transformation.py
View file

@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
) -> str | None:
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
return False
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
return False
def transform_decisions_request(

22
litellm/llms/openai/openai.py
View file

@ -16,6 +16,7 @@ from openai._base_client import make_request_options
from openai._constants import RAW_RESPONSE_HEADER
from openai._legacy_response import LegacyAPIResponse
from openai._types import RequestOptions
from openai.pagination import SyncCursorPage
from openai.types import CreateEmbeddingResponse
from openai.types.beta.assistant_deleted import AssistantDeleted
from openai.types.chat import ChatCompletion, ChatCompletionChunk
@ -40,6 +41,7 @@ from litellm.types.utils import (
LiteLLMBatch,
ModelResponse,
ModelResponseStream,
openai_batch_status,
)
from litellm.utils import (
CustomStreamWrapper,
@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM):
return response
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
return [
batch.model_copy(
update={
"object": batch.object or "batch",
"status": openai_batch_status(batch.status),
}
)
for batch in batches
]
class OpenAIBatchesAPI(BaseLLM):
"""
OpenAI methods to support for batches
@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM):
):
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
response: Final = await openai_client.batches.list(after=after, limit=limit)
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
def list_batches(
self,
@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization: str | None,
after: str | None = None,
limit: int | None = None,
client: OpenAI | None = None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM):
)
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
response: Final = openai_client.batches.list(after=after, limit=limit)
return response
if not isinstance(response, SyncCursorPage):
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
class OpenAIAssistantsAPI(BaseLLM):

86
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -38,6 +38,7 @@ from itertools import accumulate, chain, groupby, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
@ -277,6 +278,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
)
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
_TEXT_DELTA_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.refusal.delta"})
_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
@ -537,6 +539,14 @@ def _scanned_text_field(item: Mapping[str, object]) -> str:
return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
def _content_part_text_field(part: object) -> str:
return "refusal" if stream_item_field(part, "type") == "refusal" else "text"
def _has_writable_field(part: object, field: str) -> bool:
return isinstance(part, dict) or hasattr(part, field)
def _tool_output_part_text(part: object) -> str | None:
if not isinstance(part, dict):
return None
@ -993,7 +1003,7 @@ class OpenAIResponsesHandler(BaseTranslation):
for content_idx, content_item in enumerate(content):
if isinstance(content_item, dict):
# Extract text
text_str = content_item.get("text", None)
text_str = content_item.get(_content_part_text_field(content_item), None)
if text_str is not None:
texts_to_check.append(text_str)
task_mappings.append((msg_idx, int(content_idx)))
@ -1042,8 +1052,7 @@ class OpenAIResponsesHandler(BaseTranslation):
elif isinstance(content, list):
# Replace specific text item in list content
if isinstance(messages[msg_idx][field][content_idx_optional], dict):
messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
self._write_content_part_text(content[content_idx_optional], guardrail_response)
async def process_output_response(
self,
@ -1380,7 +1389,8 @@ class OpenAIResponsesHandler(BaseTranslation):
if isinstance(output_index, int) and isinstance(content_index, int)
)
if len(placeable_positions) != len(scanned_positions) or any(
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES for event in scanned_events
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES | _TEXT_DELTA_EVENT_TYPES
for event in scanned_events
):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
@ -1401,6 +1411,12 @@ class OpenAIResponsesHandler(BaseTranslation):
else:
setattr(event, field, value)
@staticmethod
def _write_content_part_text(part: object, text: str) -> None:
field: Final = _content_part_text_field(part)
if _has_writable_field(part, field):
OpenAIResponsesHandler._write_event_field(part, field, text)
@staticmethod
def _write_compaction_summary(item: object, summary: str) -> None:
encrypted_content: Final = stream_item_field(item, "encrypted_content")
@ -1416,11 +1432,12 @@ class OpenAIResponsesHandler(BaseTranslation):
) -> None:
"""Sync pre-completion stream events with the rewritten completed
response, keyed by ``(output_index, content_index)``: the first
``output_text.delta`` for a rewritten item carries the full rewritten
text and the rest are blanked, while ``output_text.done``,
``content_part.done``, and ``output_item.done`` events carry the full
rewritten text, so every event a client may read agrees with the
rewritten ``response.completed`` payload."""
``output_text.delta`` or ``refusal.delta`` for a rewritten item carries
the full rewritten text and the rest are blanked, while
``output_text.done``, ``refusal.done``, ``content_part.done``, and
``output_item.done`` events carry the full rewritten text, so every
event a client may read agrees with the rewritten ``response.completed``
payload."""
delta_replacements: Final = MappingProxyType(
{position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()}
)
@ -1436,14 +1453,14 @@ class OpenAIResponsesHandler(BaseTranslation):
if not isinstance(output_index, int) or not isinstance(content_index, int):
continue
position = (output_index, content_index)
if event_type == "response.output_text.delta" and position in delta_replacements:
if event_type in _TEXT_DELTA_EVENT_TYPES and position in delta_replacements:
self._write_event_field(event, "delta", next(delta_replacements[position]))
elif event_type == "response.output_text.done" and position in rewrites_by_position:
self._write_event_field(event, "text", rewrites_by_position[position])
elif event_type == "response.refusal.done" and position in rewrites_by_position:
self._write_event_field(event, "refusal", rewrites_by_position[position])
elif event_type == "response.content_part.done" and position in rewrites_by_position:
part = event.get("part")
if isinstance(part, dict) or hasattr(part, "text"):
self._write_event_field(part, "text", rewrites_by_position[position])
self._write_content_part_text(event.get("part"), rewrites_by_position[position])
@staticmethod
def _sync_output_item_done_event(
@ -1459,7 +1476,7 @@ class OpenAIResponsesHandler(BaseTranslation):
for (item_idx, content_idx), rewritten in rewrites_by_position.items():
if item_idx != output_index or not 0 <= content_idx < len(content):
continue
OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten)
OpenAIResponsesHandler._write_content_part_text(content[content_idx], rewritten)
def _deliver_ended_stream_tool_call_rewrites(
self,
@ -1814,7 +1831,7 @@ class OpenAIResponsesHandler(BaseTranslation):
return
# Handle both GenericResponseOutputItem and dict
content: list[OutputText] | list[dict] | None = None
content: list[OutputText | ResponseOutputRefusal] | list[dict[str, object]] | None = None
if isinstance(output_item, BaseModel):
try:
output_item_dump: Final = output_item.model_dump()
@ -1837,17 +1854,9 @@ class OpenAIResponsesHandler(BaseTranslation):
verbose_proxy_logger.debug("OpenAI Responses API: Processing output item: %s", output_item)
# Iterate through content items (list of OutputText objects)
for content_idx, content_item in enumerate(content):
# Handle both OutputText objects and dicts
if isinstance(content_item, OutputText):
text_content = content_item.text
elif isinstance(content_item, dict):
text_content = content_item.get("text")
else:
continue
if text_content:
text_content = stream_item_field(content_item, _content_part_text_field(content_item))
if isinstance(text_content, str) and text_content:
texts_to_check.append(text_content)
task_mappings.append((output_idx, int(content_idx)))
@ -1887,33 +1896,24 @@ class OpenAIResponsesHandler(BaseTranslation):
# Handle both GenericResponseOutputItem, BaseModel, and dict
if isinstance(output_item, GenericResponseOutputItem):
if output_item.content and content_idx < len(output_item.content):
content_item = output_item.content[content_idx]
if isinstance(content_item, OutputText):
content_item.text = guardrail_response
elif isinstance(content_item, dict):
content_item["text"] = guardrail_response
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
elif isinstance(output_item, BaseModel):
# Handle other Pydantic models by converting to GenericResponseOutputItem
try:
generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump())
if generic_item.content and content_idx < len(generic_item.content):
content_item = generic_item.content[content_idx]
if isinstance(content_item, OutputText):
content_item.text = guardrail_response
# Update the original response output
if hasattr(output_item, "content") and output_item.content:
original_content = output_item.content[content_idx]
if hasattr(original_content, "text"):
original_content.text = guardrail_response
if (
generic_item.content
and content_idx < len(generic_item.content)
and hasattr(output_item, "content")
and output_item.content
):
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
except Exception:
pass
elif isinstance(output_item, dict):
content = output_item.get("content", [])
if content and content_idx < len(content):
if isinstance(content[content_idx], dict):
content[content_idx]["text"] = guardrail_response
elif hasattr(content[content_idx], "text"):
content[content_idx].text = guardrail_response
self._write_content_part_text(content[content_idx], guardrail_response)
def build_block_sse_chunks(
self,

53
litellm/llms/openai/responses/transformation.py
View file

@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
tools=mapped_tools,
)
if marker_safe_input is not validated_input:
self._strip_caller_input_as_if_unmarked(
model=model,
caller_input=validated_input,
reads_breakpoints=reads_breakpoints,
maps_anthropic_tool_search=maps_anthropic_tool_search,
)
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
model=model, tools=object_schema_tools, litellm_params=litellm_params
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
return marked_input
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
def _strip_caller_input_as_if_unmarked(
self,
model: str,
caller_input: str | ResponseInputParam,
reads_breakpoints: bool,
maps_anthropic_tool_search: bool,
) -> None:
self.remove_cache_control_flag_from_input_and_tools(
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
)
def _input_with_wire_safe_prompt_cache_breakpoints(
self,
model: str,
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
the chat path. Strips Anthropic-only `cache_control` markers from
Responses API input content blocks and tools.
`filter_value_from_dict` mutates each dict in place, so the same
objects are returned.
`filter_value_from_dict` deletes the key in place and recurses, so each
item is copied first: the caller keeps its own input list and may reuse
it on a provider that does support prompt caching.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
new_input: Final = (
cast( # cast-ok: the comprehension rebuilds the input item for item
ResponseInputParam,
[without_cache_control(item) for item in input],
)
if isinstance(input, list)
else input
)
if isinstance(input, list):
for item in input:
if isinstance(item, dict):
filter_value_from_dict(cast(dict, item), "cache_control")
new_tools: Final = (
cast( # cast-ok: the comprehension rebuilds the tool list item for item
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
[without_cache_control(tool) for tool in tools],
)
if tools is not None
else None
)
if tools is not None:
for tool in tools:
if isinstance(tool, dict):
filter_value_from_dict(cast(dict, tool), "cache_control")
return input, tools
return new_input, new_tools
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):

14
litellm/llms/openai_like/README.md
View file

@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
```json
{
"your_provider": {
"supported_endpoints": ["/v1/chat/completions"],
"model_supported_endpoints": {
"model-a": ["/v1/messages"],
"model-b": ["/v1/responses"]
}
}
}
```
### How It Works
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`

77
litellm/llms/openai_like/embedding/handler.py
View file

@ -3,11 +3,16 @@
## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
import json
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse
from ..common_utils import OpenAILikeBase, OpenAILikeError
_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TRANSPORT_HEADER_NAMES: Final = frozenset(
{
"host",
"content-length",
"transfer-encoding",
"connection",
"keep-alive",
"proxy-connection",
"te",
"trailer",
"upgrade",
"expect",
}
)
_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
def _header_text(name: str, value: object) -> str | None:
if isinstance(value, bool) or not isinstance(value, str | int | float):
return None
if _HEADER_NAME.fullmatch(name) is None:
return None
return str(value).strip(" \t\r\n")
def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
caller_set: Final = {
name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES
}
texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()})
refused: Final = sorted(name for name, text in texts.items() if text is None)
if refused and not drop_params:
raise litellm.UnsupportedParamsError(
message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of "
"letters, digits or !#$%&'*+-.^_`|~ and a text or number value. "
"To drop them from the call, set `litellm.drop_params = True`.",
model=model,
)
named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None})
unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None)
if unsendable:
verbose_logger.warning(
"extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line",
unsendable,
model,
)
return MappingProxyType({name: text for name, text in named.items() if name not in unsendable})
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
def __init__(self, **kwargs):
@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
aembedding=None,
custom_endpoint: bool | None = None,
headers: dict | None = None,
drop_params: bool = False,
) -> EmbeddingResponse:
api_base, headers = self._validate_environment(
api_base=api_base,
@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
custom_endpoint=custom_endpoint,
)
model = model
filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")}
data: Final = {"model": model, "input": input, **filtered_optional_params}
data: Final = {
"model": model,
"input": input,
**{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
}
extra_headers: Final = _extra_headers_as_text(
_EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
)
key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
request_headers: Final = {
**{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
**extra_headers,
}
## LOGGING
logging_obj.pre_call(
@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
api_key=api_key,
timeout=timeout,
client=client,
headers=headers,
headers=request_headers,
)
if client is None or isinstance(client, AsyncHTTPHandler):
self.client = HTTPHandler(timeout=timeout)
@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
try:
response: Final = self.client.post(
api_base,
headers=headers,
headers=request_headers,
data=json.dumps(data),
)

25
litellm/llms/openai_like/json_loader.py
View file

@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
"""
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Final
@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
class SimpleProviderConfig:
@ -30,7 +31,10 @@ class SimpleProviderConfig:
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
data.get("special_handling", {})
)
self.supported_endpoints = data.get("supported_endpoints", [])
self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
data.get("model_supported_endpoints", {})
)
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
)
@ -90,6 +94,23 @@ class JSONProviderRegistry:
return False
return "/v1/responses" in provider.supported_endpoints
@classmethod
def supports_embeddings(cls, slug: str) -> bool:
provider: Final = cls._providers.get(slug)
if provider is None:
return False
return "/v1/embeddings" in provider.supported_endpoints
@classmethod
def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool:
provider: Final = cls._providers.get(slug)
if provider is None:
return False
model_endpoints: Final = (
provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
)
return endpoint in provider.supported_endpoints or endpoint in model_endpoints
@classmethod
def list_providers(cls) -> list:
"""List all registered provider slugs"""

12
litellm/llms/openai_like/providers.json
View file

@ -211,7 +211,7 @@
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
},
"scx-ai": {
"base_url": "https://api.scx.ai/v1",
@ -231,6 +231,16 @@
"api_base_env": "PRISM_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"hostinger_router": {
"base_url": "https://router.hostinger.com/v1",
"api_key_env": "HOSTINGER_ROUTER_API_KEY",
"api_base_env": "HOSTINGER_ROUTER_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
"model_supported_endpoints": {
"claude-sonnet-5": ["/v1/messages"],
"gpt-5.2": ["/v1/responses"]
}
},
"sail": {
"base_url": "https://api.sailresearch.com/v1",
"api_key_env": "SAIL_API_KEY",

2
litellm/llms/together_ai/chat/transformation.py
View file

@ -18,7 +18,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.prompt_templates.common_utils import strip_litellm_response_only_fields
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema

43
litellm/llms/vertex_ai/context_caching/transformation.py
View file

@ -110,38 +110,23 @@ def separate_cached_messages(
messages: list[AllMessageValues],
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
"""
Returns separated cached and non-cached messages.
Args:
messages: List of messages to be separated.
Returns:
Tuple containing:
- cached_messages: List of cached messages.
- non_cached_messages: List of non-cached messages.
Returns the first continuous block of cached messages, plus every system message once
any message is cached, since generateContent rejects a system_instruction beside a
cachedContent, and the remaining messages.
"""
cached_messages: list[AllMessageValues] = []
non_cached_messages: list[AllMessageValues] = []
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
if not filtered_messages:
return [], messages
# Extract cached messages and their indices
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
for idx, message in enumerate(messages):
if is_cached_message(message=message):
filtered_messages.append((idx, message))
# Validate only one block of continuous cached messages
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
# Separate messages based on the block of cached messages
if filtered_messages and last_continuous_block_idx is not None:
first_cached_idx: Final = filtered_messages[0][0]
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
else:
non_cached_messages = messages
return cached_messages, non_cached_messages
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
goes_in_cache: Final = tuple(
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
)
return (
[message for message, cached in zip(messages, goes_in_cache) if cached],
[message for message, cached in zip(messages, goes_in_cache) if not cached],
)
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:

30
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -31,9 +31,8 @@ from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
from litellm.litellm_core_utils.prompt_templates.factory import (
encode_tool_call_id_with_signature,
)
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -2091,6 +2090,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
parts: list[HttpxPartType],
cumulative_tool_call_idx: int,
is_function_call: bool | None,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[
ChatCompletionToolCallFunctionChunk | None,
list[ChatCompletionToolCallChunk] | None,
@ -2129,9 +2129,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# back on the matching `functionResponse`.
if gemini_call_id:
_tool_response_chunk["id"] = gemini_call_id
# Embed thought signature in ID for OpenAI client compatibility
if thought_signature:
_tool_response_chunk["provider_specific_fields"] = {"thought_signature": thought_signature}
if thought_signature and thought_signature_in_tool_call_id:
_tool_response_chunk["id"] = encode_tool_call_id_with_signature(
_tool_response_chunk["id"] or "", thought_signature
)
@ -2793,6 +2793,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
model_response: Union[ModelResponse, "ModelResponseStream"],
standard_optional_params: dict,
cumulative_tool_call_index: int = 0,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[list[dict], list[dict], list, list, int]:
"""
Helper method to process candidates and extract metadata
@ -2895,6 +2896,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
parts=candidate["content"]["parts"],
cumulative_tool_call_idx=cumulative_tool_call_index,
is_function_call=is_function_call(standard_optional_params),
thought_signature_in_tool_call_id=thought_signature_in_tool_call_id,
)
if "logprobsResult" in candidate:
@ -2978,6 +2980,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
model_response: Union[ModelResponse, "ModelResponseStream"],
standard_optional_params: dict[str, object], # mutable-ok: mirrors override contract
cumulative_tool_call_index: int = 0,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[ # mutable-ok: mirrors override contract
list[dict[str, object]],
list[dict[str, object]],
@ -2986,7 +2989,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
int,
]:
return cls._process_candidates(
_candidates, model_response, standard_optional_params, cumulative_tool_call_index
_candidates,
model_response,
standard_optional_params,
cumulative_tool_call_index,
thought_signature_in_tool_call_id,
)
def transform_response(
@ -3069,7 +3076,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
safety_ratings,
citation_metadata,
_, # cumulative_tool_call_index not needed in non-streaming
) = VertexGeminiConfig._process_candidates(_candidates, model_response, logging_obj.optional_params)
) = VertexGeminiConfig._process_candidates(
_candidates,
model_response,
logging_obj.optional_params,
thought_signature_in_tool_call_id=embeds_thought_signature_in_tool_call_id(
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
),
)
usage: Final = VertexGeminiConfig._calculate_usage(
completion_response=completion_response,
@ -3905,6 +3919,9 @@ class ModelResponseIterator:
self.logging_obj = logging_obj
self.response_headers = response_headers or {}
self.is_function_call = check_is_function_call(logging_obj)
self.thought_signature_in_tool_call_id: Final = embeds_thought_signature_in_tool_call_id(
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
)
self.cumulative_tool_call_index: int = 0
self.has_seen_tool_calls: bool = False
self._resume = resume
@ -3969,6 +3986,7 @@ class ModelResponseIterator:
model_response,
self.logging_obj.optional_params,
cumulative_tool_call_index=self.cumulative_tool_call_index,
thought_signature_in_tool_call_id=self.thought_signature_in_tool_call_id,
)
# Track whether tool_calls have been seen across streaming chunks.

3
litellm/llms/watsonx/chat/transformation.py
View file

@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
"""
import copy
from typing import Final
from litellm import verbose_logger
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
_tools = non_default_params.pop("tools", None)
if _tools is not None:
# remove 'additionalProperties' from tools
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
# remove 'strict' from tools
_tools = remove_strict_from_schema(_tools)
if _tools is not None:

4
litellm/llms/xai/chat/transformation.py
View file

@ -1,3 +1,4 @@
import copy
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import Any, Final
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
tools = [
filtered_tool
for filtered_tool in (
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
for tool in value
)
if filtered_tool is not None
]

134
litellm/main.py
View file

@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin
from concurrent import futures
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from copy import deepcopy
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
@ -82,8 +83,8 @@ from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_litellm_params import (
AWS_CREDENTIAL_KWARGS_KEYS,
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
OPTIONAL_KWARGS_KEYS,
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
SUPPLEMENTAL_KWARGS_KEYS,
InvalidControlOption,
parse_control_options,
with_control_options,
@ -113,6 +114,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
messages_without_pdf_file_parts,
)
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.reasoning_effort_capability import (
model_group_from_metadata,
reasoning_effort_needs_allowing,
resolve_chat_reasoning,
)
from litellm.litellm_core_utils.request_timeout_resolver import (
get_configured_request_timeout,
)
@ -144,11 +150,6 @@ from litellm.llms.vertex_ai.common_utils import (
)
from litellm.realtime_api.main import realtime_health_check
from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
from litellm.router_utils.reasoning_effort_capability import (
model_group_from_metadata,
reasoning_effort_needs_allowing,
resolve_chat_reasoning,
)
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.completion import (
@ -157,7 +158,7 @@ from litellm.types.completion import (
)
from litellm.types.litellm_params import ControlOptions, RetryStrategy
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_strictly
from litellm.types.utils import (
AgenticLoopParams,
BaseUrlRoutingHint,
@ -399,6 +400,49 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
return assert_never(control)
@dataclass(frozen=True, slots=True)
class _SupplementalProviderParams:
params: GenericLiteLLMParams
dropped_keys: frozenset[str] = frozenset()
@dataclass(frozen=True, slots=True)
class _InvalidSupplementalParams:
message: str
param: str
def _parse_supplemental_provider_params(
kwargs: Mapping[str, object],
) -> _SupplementalProviderParams | _InvalidSupplementalParams:
supplemental: Final = {key: kwargs[key] for key in SUPPLEMENTAL_KWARGS_KEYS if key in kwargs}
try:
return _SupplementalProviderParams(params=validate_litellm_params_strictly(GenericLiteLLMParams, supplemental))
except ValidationError as error:
invalid: Final = tuple(detail for detail in error.errors() if detail["loc"])
invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid)
if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
return _SupplementalProviderParams(
params=GenericLiteLLMParams.model_validate(
{key: value for key, value in supplemental.items() if key not in invalid_keys}
),
dropped_keys=invalid_keys,
)
return _InvalidSupplementalParams(
message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid),
param=", ".join(sorted(invalid_keys)),
)
def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams:
parsed: Final = _parse_supplemental_provider_params(kwargs)
if isinstance(parsed, _InvalidSupplementalParams):
raise litellm.BadRequestError(
message=parsed.message, model=model, llm_provider=None, body={"param": parsed.param}
)
return parsed
PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"})
PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"})
@ -830,21 +874,17 @@ async def acompletion(
"enable_json_schema_validation": enable_json_schema_validation,
}
if custom_llm_provider is None:
_supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects
"dict[str, object]",
{
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
"extra_headers": extra_headers,
},
)
request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs
_, custom_llm_provider, _, _ = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=cast( # cast-ok: api_base arrives as a str in the request kwargs
"str | None", kwargs.get("api_base")
"str | None", request_kwargs.get("api_base")
)
or base_url,
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
litellm_params=_supplemental_provider_params(
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
).params,
)
if fallbacks is not None:
@ -4382,6 +4422,7 @@ _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
@ -5625,6 +5666,11 @@ def completion(
######### unpacking kwargs #####################
args: Final = _locals_snapshot(locals())
supplemental_params: Final = _supplemental_provider_params(
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
)
for dropped_key in supplemental_params.dropped_keys:
kwargs.pop(dropped_key)
# Set by the responses->completion fallback so completion() does not bridge
# back to the Responses API: that round-trip mutually recurses forever for a
@ -5850,21 +5896,14 @@ def completion(
if deployment_id is not None: # azure llms
model = deployment_id
custom_llm_provider = "azure"
_supplemental_provider_params: Final = {
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
"extra_headers": extra_headers,
}
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
requested_model: Final = model
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
litellm_params=(
GenericLiteLLMParams.model_validate(_supplemental_provider_params)
if _supplemental_provider_params
else None
),
litellm_params=supplemental_params.params,
)
_record_agentic_loop_params(
logging_obj=logging,
@ -5884,7 +5923,6 @@ def completion(
api_base=api_base,
deployment_model_info=deployment_model_info,
)
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
drop_unsupported_reasoning: Final = (
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
)
@ -6220,6 +6258,7 @@ def completion(
"github_copilot_user_session",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
"fireworks_forward_user_id",
"thought_signature_in_tool_call_id",
"responses_api_version",
"allowed_openai_params",
)
@ -6809,6 +6848,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
)
def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]:
if not configured:
return {}
if isinstance(configured, Mapping):
return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON
try:
return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON
except (TypeError, ValueError):
if drop_params:
return {}
raise litellm.UnsupportedParamsError(
message=f"{param} must be a JSON object of header names to values. "
"To drop it from the call, set `litellm.drop_params = True`.",
model=model,
)
# fmt: off
# Overload for when aembedding=True (returns coroutine)
@ -6923,11 +6979,10 @@ def embedding(
azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None)
aembedding: Final[bool | None] = kwargs.get("aembedding", None)
extra_headers: Final = kwargs.get("extra_headers", None)
headers = kwargs.get("headers", None) or extra_headers
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
headers = {
**_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params),
**_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params),
}
# Inject proxy auth headers if configured
if litellm.proxy_auth is not None:
try:
@ -7091,7 +7146,7 @@ def embedding(
# set API KEY
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
if headers is not None and headers != {}:
if headers:
optional_params["extra_headers"] = headers
requested_encoding_format: Final = (
@ -7183,14 +7238,15 @@ def embedding(
custom_llm_provider == "openai_like"
or custom_llm_provider == "llamafile"
or custom_llm_provider == "lm_studio"
or JSONProviderRegistry.supports_embeddings(custom_llm_provider)
):
api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE")
# set API KEY
if api_key is None:
api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider):
api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
if headers is not None and headers != {}:
if headers:
optional_params["extra_headers"] = headers
## EMBEDDING CALL
@ -7205,10 +7261,9 @@ def embedding(
optional_params=optional_params,
client=client,
aembedding=aembedding,
drop_params=drop_unsupported_params,
)
elif custom_llm_provider == "oci":
if headers is None:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,
@ -7233,11 +7288,6 @@ def embedding(
or litellm.api_key
)
# Use the merged headers variable (already merged at the top of the function)
# Don't overwrite it with just extra_headers
if headers is None:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,

103
litellm/model_prices_and_context_window_backup.json
View file

@ -25312,6 +25312,38 @@
"cache_creation_input_token_cost": 1e-06,
"prompt_cache_min_tokens": 2048
},
"eu.anthropic.claude-haiku-4-5-20251001-v1:0": {
"cache_creation_input_token_cost": 1.375e-06,
"cache_creation_input_token_cost_above_1hr": 2.2e-06,
"cache_read_input_token_cost": 1.1e-07,
"input_cost_per_token": 1.1e-06,
"deprecation_date": "2026-10-15",
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5.5e-06,
"source": "https://aws.amazon.com/bedrock/pricing/",
"supports_assistant_prefill": true,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_native_structured_output": true,
"supports_parallel_tool_use_config": true,
"prompt_cache_min_tokens": 4096,
"input_cost_per_token_batches": 5.5e-07,
"output_cost_per_token_batches": 2.75e-06,
"token_count_model": {
"model": "anthropic.claude-haiku-4-5-20251001-v1:0",
"litellm_provider": "bedrock"
}
},
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06,
"litellm_provider": "bedrock",
@ -39665,6 +39697,57 @@
"/v1/images/generations"
]
},
"hostinger_router/gpt-5.2": {
"cache_read_input_token_cost": 0.00000021,
"input_cost_per_token": 0.0000021,
"litellm_provider": "hostinger_router",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.0000168,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_vision": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.3e-06,
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
"cache_read_input_token_cost": 2.64e-07,
"input_cost_per_token": 2.64e-06,
"litellm_provider": "hostinger_router",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.32e-05,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_response_schema": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-large": {
"input_cost_per_token": 1.716e-07,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-small": {
"input_cost_per_token": 3e-08,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "hyperbolic",
@ -51365,6 +51448,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -51393,6 +51481,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52139,6 +52232,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52244,6 +52342,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

26
litellm/provider_endpoints_support_backup.json
View file

@ -1087,6 +1087,32 @@
"generateContent": false
}
},
"hostinger_router": {
"display_name": "Hostinger Router (`hostinger_router`)",
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"crusoe": {
"display_name": "Crusoe (`crusoe`)",
"url": "https://docs.litellm.ai/docs/providers/crusoe",

23
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -24268,7 +24268,14 @@
"properties": {
"content": {
"items": {
"$ref": "#/components/schemas/OutputText"
"anyOf": [
{
"$ref": "#/components/schemas/OutputText"
},
{
"$ref": "#/components/schemas/ResponseOutputRefusal"
}
]
},
"title": "Content",
"type": "array"
@ -58015,7 +58022,7 @@
"paths": {
"/toolset/{toolset_name}/mcp": {
"delete": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete",
"parameters": [
{
@ -58054,7 +58061,7 @@
]
},
"get": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get",
"parameters": [
{
@ -58093,7 +58100,7 @@
]
},
"head": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head",
"parameters": [
{
@ -58132,7 +58139,7 @@
]
},
"options": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options",
"parameters": [
{
@ -58171,7 +58178,7 @@
]
},
"patch": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch",
"parameters": [
{
@ -58210,7 +58217,7 @@
]
},
"post": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post",
"parameters": [
{
@ -58249,7 +58256,7 @@
]
},
"put": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"parameters": [
{

3
litellm/proxy/agent_endpoints/a2a_endpoints.py
View file

@ -431,6 +431,7 @@ async def _handle_stream_message(
proxy_server_request: dict[str, object] | None = None,
*,
agent_extra_headers: dict[str, str] | None = None,
agent_name: str | None = None,
user_api_key_dict: UserAPIKeyAuth | None = None,
request_data: dict[str, object] | None = None,
proxy_logging_obj: ProxyLogging | None = None,
@ -504,6 +505,7 @@ async def _handle_stream_message(
metadata=metadata,
proxy_server_request=proxy_server_request,
agent_extra_headers=agent_extra_headers,
agent_name=agent_name,
)
if (
@ -932,6 +934,7 @@ async def invoke_agent_a2a(
metadata=data.get("metadata", {}),
proxy_server_request=data.get("proxy_server_request"),
agent_extra_headers=agent_extra_headers,
agent_name=str(agent_name),
user_api_key_dict=user_api_key_auth_for_call,
request_data=data,
proxy_logging_obj=proxy_logging_obj,

37
litellm/proxy/auth/auth_checks.py
View file

@ -167,6 +167,7 @@ from litellm.repositories.table_repositories import (
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.router import Router
from litellm.router_api import RegistryReader
from litellm.types.proxy.auth.auth_checks import UserNotFoundError
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
from litellm.types.router import DeploymentTypedDict
@ -525,7 +526,7 @@ def _log_budget_lookup_failure(entity: str, error: Exception) -> None:
)
def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
def _get_router_zero_cost_cache(llm_router: RegistryReader) -> dict[str, bool] | None:
"""
Return the router's per-instance zero-cost cache, or ``None`` for objects
that don't expose one (e.g. ``MagicMock`` stand-ins in unit tests).
@ -541,7 +542,7 @@ def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
return cache if isinstance(cache, dict) else None
def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None) -> bool:
def is_model_cost_zero(model: str | list[str] | None, llm_router: RegistryReader | None) -> bool:
"""
Check if a model has zero cost (no configured pricing).
@ -667,7 +668,7 @@ _NO_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({})
_TEAM_GRANT_RELATIONS: Final[Mapping[str, object]] = MappingProxyType({"litellm_model_table": True})
def _deployments_served_for(model: str, llm_router: "Router") -> Sequence[DeploymentTypedDict]:
def _deployments_served_for(model: str, llm_router: RegistryReader) -> Sequence[DeploymentTypedDict]:
"""
The deployments a request to ``model`` routes to, for the gates that waive budget checks.
@ -4842,7 +4843,7 @@ async def get_skill_names_from_access_groups(
def _resolve_all_team_model_sentinel_for_auth_check(
models: list[str],
llm_router: Router | None,
llm_router: RegistryReader | None,
team_id: str | None,
) -> list[str]:
if SpecialModelNames.all_team_models.value not in models or team_id is None or llm_router is None:
@ -4856,7 +4857,7 @@ def _resolve_all_team_model_sentinel_for_auth_check(
def check_model_access_helper(
model: str,
llm_router: Router | None,
llm_router: RegistryReader | None,
models: list[str],
team_model_aliases: dict[str, str] | None = None,
team_id: str | None = None,
@ -4907,7 +4908,7 @@ _check_model_access_helper: Final = check_model_access_helper
def _owning_team_names_for_team_deployment(
model: str,
llm_router: Router | None,
llm_router: RegistryReader | None,
team_id: str | None,
team_model_aliases: Mapping[str, str] | None,
) -> tuple[str, ...]:
@ -4925,7 +4926,7 @@ def _owning_team_names_for_team_deployment(
def _can_object_call_model(
model: str | list[str],
llm_router: Router | None,
llm_router: RegistryReader | None,
models: list[str],
team_model_aliases: dict[str, str] | None = None,
team_id: str | None = None,
@ -5052,7 +5053,7 @@ def _resolve_team_alias(
def _live_team_alias_target(
model: str, team_model_aliases: Mapping[str, str], team_id: str | None, llm_router: Router | None
model: str, team_model_aliases: Mapping[str, str], team_id: str | None, llm_router: RegistryReader | None
) -> str:
target: Final = team_model_aliases.get(model)
if target is None:
@ -5539,7 +5540,7 @@ async def can_key_call_model(
model: str | list[str],
llm_model_list: Sequence[object] | None,
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router | None,
llm_router: RegistryReader | None,
prisma_client: DatabaseClient | None = None,
) -> Literal[True]:
"""
@ -5618,7 +5619,7 @@ async def can_key_call_resolved_model(
model: str,
llm_model_list: Sequence[object] | None,
valid_token: UserAPIKeyAuth,
llm_router: litellm.Router | None,
llm_router: RegistryReader | None,
) -> None:
from litellm.proxy.proxy_server import (
prisma_client,
@ -5766,7 +5767,7 @@ def can_org_access_model(
async def can_team_access_model(
model: str | list[str],
team_object: LiteLLM_TeamTable | None,
llm_router: Router | None,
llm_router: RegistryReader | None,
team_model_aliases: dict[str, str] | None = None,
key_model_aliases: Mapping[str, str] | None = None,
prisma_client: DatabaseClient | None = None,
@ -5869,7 +5870,7 @@ async def key_access_group_grants_model(
model: str | list[str],
valid_token: UserAPIKeyAuth | None,
team_object: LiteLLM_TeamTable | None,
llm_router: Router | None,
llm_router: RegistryReader | None,
key_model_aliases: Mapping[str, str] | None = None,
) -> bool:
"""
@ -5905,7 +5906,7 @@ _key_access_group_grants_model: Final = key_access_group_grants_model
def can_project_access_model(
model: str | list[str],
project_object: LiteLLM_ProjectTable,
llm_router: Router | None,
llm_router: RegistryReader | None,
key_model_aliases: Mapping[str, str] | None = None,
) -> Literal[True]:
"""
@ -5926,7 +5927,7 @@ def can_project_access_model(
def can_customer_access_model(
model: str | list[str],
end_user_object: LiteLLM_EndUserTable,
llm_router: Router | None,
llm_router: RegistryReader | None,
valid_token: UserAPIKeyAuth | None,
) -> Literal[True]:
team_model_aliases: Final = team_model_aliases_for_auth_check(valid_token) if valid_token is not None else None
@ -6089,7 +6090,7 @@ async def _team_grants_personal_key_model(
model: str,
team_object: LiteLLM_TeamTableCachedObj,
valid_token: UserAPIKeyAuth,
llm_router: Router | None,
llm_router: RegistryReader | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
@ -6127,7 +6128,7 @@ async def _loaded_teams_allow_model(
model: str,
teams: tuple[LiteLLM_TeamTableCachedObj | None, ...],
valid_token: UserAPIKeyAuth,
llm_router: Router | None,
llm_router: RegistryReader | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
@ -6393,7 +6394,7 @@ async def can_personal_key_call_model_via_teams(
model: str | Sequence[str],
user_object: LiteLLM_UserTable | None,
valid_token: UserAPIKeyAuth,
llm_router: Router | None,
llm_router: RegistryReader | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
@ -7185,7 +7186,7 @@ async def check_team_member_model_access(
model: str | list[str],
team_object: LiteLLM_TeamTable,
valid_token: UserAPIKeyAuth,
llm_router: Router | None,
llm_router: RegistryReader | None,
prisma_client: Optional["PrismaClient"],
user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,

10
litellm/proxy/auth/fallback_budget.py
View file

@ -44,7 +44,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
_is_model_cost_zero, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
is_model_cost_zero, # pyright: ignore[reportPrivateUsage] # the zero-cost predicate the auth-time budget checks use; no public equivalent
)
from litellm.router import Router
from litellm.router_api import RegistryReader, RoutingContext
from litellm.types.llms.base import LiteLLMBaseModel
@ -118,7 +118,7 @@ async def is_token_within_budget_for_model(
*,
model: str,
valid_token: UserAPIKeyAuth,
llm_router: Router,
llm_router: RegistryReader,
team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None,
) -> bool:
"""
@ -187,19 +187,19 @@ class RouterFallbackBudgetCheck:
is_enforced: Callable[[], bool]
team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool:
if not self.is_enforced():
return True
valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
if valid_token is None:
return True
budget_models: Final = tuple(dict.fromkeys((model, llm_router.registry().resolve(model))))
budget_models: Final = tuple(dict.fromkeys((model, ctx.registry().resolve(model))))
for budget_model in budget_models:
try:
if not await is_token_within_budget_for_model(
model=budget_model,
valid_token=valid_token,
llm_router=llm_router,
llm_router=ctx,
team_member_model_budget_limiter=self.team_member_model_budget_limiter,
):
return False

8
litellm/proxy/auth/fallback_model_access.py
View file

@ -18,7 +18,7 @@ from pydantic import ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, can_token_call_search_tool
from litellm.router import Router
from litellm.router_api import RegistryReader, RoutingContext
from litellm.search import asearch
from litellm.types.llms.base import LiteLLMBaseModel
@ -31,7 +31,7 @@ class _FallbackAccessSettings(LiteLLMBaseModel):
enforce_fallback_model_access: bool = True
async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: Router) -> bool:
async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: RegistryReader) -> bool:
try:
await can_key_call_resolved_model(
model=model,
@ -98,7 +98,7 @@ class RouterFallbackAccessCheck:
is_enforced: Callable[[], bool]
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool:
valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
if valid_token is None:
return True
@ -106,7 +106,7 @@ class RouterFallbackAccessCheck:
return await is_search_tool_authorized_for_token(search_tool_name=model, valid_token=valid_token)
if not self.is_enforced():
return True
return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=llm_router)
return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=ctx)
router_fallback_access_check: Final = RouterFallbackAccessCheck(is_enforced=_enforced_by_general_settings)

6
litellm/proxy/auth/model_checks.py
View file

@ -14,13 +14,13 @@ from typing import Final, TypeVar
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.responses.main import strip_responses_routing_prefix
from litellm.router import Router
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.router_api import fallback_chain
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
@ -708,7 +708,7 @@ def get_all_fallbacks(
try:
# Use existing function to get fallback model group
fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks_config, model_group=model)
fallback_model_group, _ = fallback_chain(fallbacks=fallbacks_config, model_group=model)
if fallback_model_group is None:
return []

26
litellm/proxy/auth/user_api_key_auth.py
View file

@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model
from litellm.models.end_user import LiteLLM_EndUserTable
from litellm.proxy._types import *
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
@ -182,13 +183,7 @@ from litellm.proxy.utils import (
from litellm.repositories.table_repositories import JWTKeyMappingRepository, TeamMembershipRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
from litellm.router_utils.fallback_event_handlers import (
PROVIDER_SCOPED_RESOURCE_KEYS,
fallback_lookup_groups,
fallbacks_disabled_for_request,
get_fallback_model_group_for_lookup_groups,
)
from litellm.router_api import fallbacks_disabled, references_provider_scoped_resource, request_fallback_chain
from litellm.secret_managers.main import get_secret_bool
from litellm.types.services import ServiceTypes
@ -811,11 +806,7 @@ async def _check_team_member_model_budget_for_model(
model=model_name,
)
except litellm.BudgetExceededError:
if (
request_data.get("model") != model_name
or llm_router is None
or fallbacks_disabled_for_request(kwargs=request_data)
):
if request_data.get("model") != model_name or llm_router is None or fallbacks_disabled(request_data):
raise
fallback_succeeded: Final = await _try_team_member_model_budget_fallback(
user_id=user_id,
@ -857,9 +848,9 @@ async def _try_team_member_model_budget_fallback(
user_api_key_cache: UserApiKeyCache | None = None,
proxy_logging_obj: ProxyLogging | None = None,
) -> bool:
if request.headers.get("content-type", "").lower().startswith("multipart/form-data") or any(
request_data.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS
):
if request.headers.get("content-type", "").lower().startswith(
"multipart/form-data"
) or references_provider_scoped_resource(request_data):
return False
from litellm.proxy import proxy_server
@ -896,10 +887,7 @@ async def _try_team_member_model_budget_fallback(
if router_fallbacks is None:
return False
fallback_models, _ = get_fallback_model_group_for_lookup_groups(
fallbacks=list(router_fallbacks),
lookup_groups=fallback_lookup_groups(kwargs=request_data, model_group=model_name),
)
fallback_models, _ = request_fallback_chain(list(router_fallbacks), request_data, model_name)
fallback_targets: Final[Sequence[object]] = fallback_models or ()
fallback_candidates: Final[tuple[str, ...]] = tuple(
model for target in fallback_targets if (model := _budget_fallback_target_model(target)) is not None

2
litellm/proxy/bug_report_config.py
View file

@ -19,8 +19,8 @@ from litellm.litellm_core_utils.bug_report import (
build_bug_report,
build_environment_report,
)
from litellm.litellm_core_utils.routing_groups import VALID_ROUTING_STRATEGIES
from litellm.proxy._types import ConfigGeneralSettings
from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES
from litellm.types.caching import LiteLLMCacheType
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, SupportedGuardrailIntegrations
from litellm.types.secret_managers.main import KeyManagementSystem

18
litellm/proxy/common_request_processing.py
View file

@ -53,6 +53,7 @@ from litellm.constants import (
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.otel.runtime import phase_event
from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.litellm_core_utils.bug_report import (
allowlisted,
bug_report_notice,
@ -81,6 +82,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
get_response_headers,
)
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.served_output_texts import (
record_served_output_texts,
@ -146,8 +148,6 @@ from litellm.responses.streaming_iterator import partial_responses_incomplete_ev
from litellm.router import Router
from litellm.router_api import RoutingContext
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
from litellm.types.extraction import ExtractionResponse
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.router import RouterRateLimitError
@ -2565,7 +2565,7 @@ class ProxyBaseLLMRequestProcessing:
aliased_model: Final = self.data.get("model")
if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str):
self.data.update(
llm_router.priority_header_update(
llm_router.health().priority_header_fields(
model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
)
)
@ -2831,12 +2831,9 @@ class ProxyBaseLLMRequestProcessing:
@staticmethod
def _resolve_fallback_models(model: str, fallbacks: list) -> list[str] | None:
from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
from litellm.router_api import fallback_chain
fallback_model_group, generic_fallback_idx = get_fallback_model_group(
fallbacks=fallbacks,
model_group=model,
)
fallback_model_group, generic_fallback_idx = fallback_chain(fallbacks, model)
if fallback_model_group is not None:
return fallback_model_group
return fallbacks[generic_fallback_idx]["*"] if generic_fallback_idx is not None else None
@ -4190,8 +4187,8 @@ class ProxyBaseLLMRequestProcessing:
HiddenParamsAsyncIteratorWrapper, so class sniffing runs on the
unwrapped inner iterator.
"""
from litellm.litellm_core_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.router_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
unwrapped: Final = response.inner if isinstance(response, HiddenParamsAsyncIteratorWrapper) else response
@ -5034,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing:
if stream_usage is None:
return None
service_tier: Final = obj.get("service_tier")
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
ModelResponse(usage=stream_usage),
ModelResponse(model=served_model, usage=stream_usage),
model_name,
service_tier if isinstance(service_tier, str) else None,
litellm_logging_obj,

13
litellm/proxy/common_utils/cache_aware_routing.py
View file

@ -16,6 +16,7 @@ from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_f
from litellm.llms.anthropic.cache_aware_routing import AnthropicCacheRouting, TokenCounter
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
from litellm.router_deprecated import renamed_router_param
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import Deployment, PreRoutingHookResponse
@ -23,6 +24,7 @@ from litellm.types.utils import StandardLoggingRoutingDecision
if TYPE_CHECKING:
from litellm.router import Router
from litellm.router_api import RoutingContext
_MESSAGES: Final = TypeAdapter(list[Mapping[str, object]])
_MAPPING: Final = TypeAdapter(Mapping[str, object])
@ -231,7 +233,7 @@ async def select_cached_model(
async def _choose_cached_model(
*,
router: Router,
ctx: RoutingContext,
config: ComplexityRouterConfig,
params_for_model: Callable[[str, str], Mapping[str, object]],
response: PreRoutingHookResponse | None,
@ -249,8 +251,10 @@ async def _choose_cached_model(
)
from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
router: Final = proxy_server.llm_router
if (
proxy_server.llm_router is not router
router is None
or not ctx.belongs_to(router)
or router.registry().plugins("routing")
or has_request_transforms()
or compaction_pending(request_kwargs)
@ -318,9 +322,10 @@ async def _choose_cached_model(
)
@renamed_router_param("router")
async def choose_cached_model(
*,
router: Router,
ctx: RoutingContext,
config: ComplexityRouterConfig,
params_for_model: Callable[[str, str], Mapping[str, object]],
response: PreRoutingHookResponse | None,
@ -332,7 +337,7 @@ async def choose_cached_model(
try:
return await asyncio.wait_for(
_choose_cached_model(
router=router,
ctx=ctx,
config=config,
params_for_model=params_for_model,
response=response,

2
litellm/proxy/common_utils/credential_hydration.py
View file

@ -12,10 +12,10 @@ from types import MappingProxyType
from typing import Final, cast # noqa: TID251 # narrows untyped credential values at the decrypt boundary
import litellm
from litellm.litellm_core_utils.clientside_credential_handler import clientside_credential_keys
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.proxy.utils import PrismaClient
from litellm.repositories.credentials_repository import CredentialsRepository
from litellm.router_utils.clientside_credential_handler import clientside_credential_keys
from litellm.types.router import (
GenericLiteLLMParams,
server_owned_wif_fields_present,

2
litellm/proxy/common_utils/model_listing_utils.py
View file

@ -21,7 +21,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import CLAUDE_CODE_CLIENT, CLAUDE_CODE_PICKER_PATTERN, GATEWAY_CLIENT_HEADER
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
if TYPE_CHECKING:
from litellm.router import Router

83
litellm/proxy/db/accounting_ledger.py
View file

@ -31,6 +31,13 @@ class AccountingConflictError(ValueError):
pass
@dataclass(frozen=True, slots=True)
class UsageWrite:
event: InboundUsageEvent | OutboundUsageEvent
envelope: str
targets: tuple[DeliveryTarget, ...] = ()
_Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"]
@ -107,6 +114,35 @@ SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_ev
_LOCK_EVENT: Final = """
SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE
"""
_INSERT_USAGE_BATCH: Final = """
INSERT INTO "LiteLLM_AccountingEvent"
(event_id, kind, schema_version, payload_hash, occurred_at, payload)
SELECT event_id, kind::"AccountingEventKind", 1, payload_hash, occurred_at::timestamp, payload
FROM jsonb_to_recordset($1::jsonb) AS records(
event_id text, kind text, payload_hash text, occurred_at text, payload jsonb)
ORDER BY event_id
ON CONFLICT (event_id) DO NOTHING RETURNING event_id
"""
_LOCK_USAGE_BATCH: Final = """
SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent"
WHERE event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb))
ORDER BY event_id FOR UPDATE
"""
_LOAD_USAGE_BATCH: Final = """
SELECT event.payload, ARRAY(
SELECT target::text FROM "LiteLLM_AccountingDelivery" delivery
WHERE delivery.event_id = event.event_id ORDER BY target::text
) AS targets
FROM "LiteLLM_AccountingEvent" event
WHERE event.event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb))
AND event.kind = 'outbound_usage' AND event.schema_version = 1
"""
_INSERT_USAGE_DELIVERIES: Final = """
INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version)
SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1
FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb)
CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination
"""
_COMPLETED_FAMILY: Final = """
root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
AND root.recorded_at < $1::timestamp
@ -199,6 +235,53 @@ class AccountingLedger:
return cls(transaction)
async def insert_usage_batch(self, writes: tuple[UsageWrite, ...]) -> None:
records: Final = tuple(_prepare(write.event, write.targets, write.envelope, 1, 1) for write in writes)
if not records:
return
unique: Final = {record.event_id: record for record in records}
if any(record.payload_hash != unique[record.event_id].payload_hash for record in records):
raise AccountingConflictError("Batch contains conflicting copies of a usage identity")
batch: Final = _json(
tuple(
{
"event_id": record.event_id,
"kind": record.kind.value,
"payload_hash": record.payload_hash,
"occurred_at": record.occurred_at.astimezone(timezone.utc).isoformat(),
"payload": _JSON.validate_json(record.payload),
}
for record in unique.values()
)
)
async with self._transaction() as db:
inserted: Final = _STRINGS.validate_python(await db.query_raw(_INSERT_USAGE_BATCH, batch))
accepted: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_USAGE_BATCH, _json(tuple(unique))))
if len(accepted) != len(unique) or any(
row["payload_hash"] != unique[row["event_id"]].payload_hash for row in accepted
):
raise AccountingConflictError("Usage identity conflicts with accepted content or delivery set")
deliveries: Final = _json(
tuple({"event_id": row["event_id"], "targets": unique[row["event_id"]].targets} for row in inserted)
)
await db.execute_raw(_INSERT_USAGE_DELIVERIES, deliveries)
async def load_usage_batch(self, event_ids: tuple[str, ...]) -> tuple[UsageWrite, ...]:
if not event_ids:
return ()
async with self._transaction() as db:
rows: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(
await db.query_raw(_LOAD_USAGE_BATCH, _json(event_ids))
)
return tuple(
UsageWrite(
OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
_json(row["payload"]),
TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
)
for row in rows
)
async def insert_event(
self,
event: AccountingEvent,

18
litellm/proxy/db/exception_handler.py
View file

@ -72,6 +72,22 @@ def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]:
return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException))
def _prisma_error_code(e: Exception) -> str | None:
import prisma.engine.errors
if isinstance(e, _exception_types(prisma.errors.DataError)):
code: Final[object] = getattr(e, "code", None)
return code if isinstance(code, str) else None
if not isinstance(e, _exception_types(prisma.engine.errors.EngineRequestError)):
return None
try:
body: Final = _DATABASE_ERROR_META.validate_json(str(e).partition(": ")[2])
except ValidationError:
return None
engine_code: Final = body.get("error_code")
return engine_code if isinstance(engine_code, str) else None
class PrismaDBExceptionHandler:
"""
Class to handle DB Exceptions or Connection Errors
@ -206,6 +222,8 @@ class PrismaDBExceptionHandler:
return True
if PrismaDBExceptionHandler.is_database_capacity_error(e):
return False
if _prisma_error_code(e) == "P1017":
return True
if isinstance(e, _exception_types(prisma.errors.PrismaError)):
error_message: Final = str(e).lower()
connection_keywords: Final = (

96
litellm/proxy/guardrails/anthropic_sse.py
View file

@ -12,18 +12,31 @@ from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields
from litellm.llms.anthropic.messages.stream_assembly import (
anthropic_message_delta_fields,
anthropic_sse_error_events,
anthropic_sse_event_type,
anthropic_sse_event_types,
anthropic_sse_started_blocks,
anthropic_sse_stopped_block_index,
anthropic_sse_unfinished_blocks,
)
from litellm.llms.anthropic.messages.stream_assembly import (
assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
has_unreadable_json_event as has_unreadable_json_event, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.utils import Choices, ModelResponse
@ -101,6 +114,48 @@ def _rewritten_message_delta_fields(
)
_MESSAGE_END_EVENT_TYPES: Final = frozenset({"message_delta", "message_stop"})
def _text_block_left_open(rewritten_chunks: Sequence[bytes], source_chunks: Sequence[object]) -> int | None:
"""Index of the rewritten text block to leave open when the failed upstream never stopped its text.
Assembly keeps a cut-off text block and drops every other cut-off block, and the round trip
merges all text into one block, so that block carries text the upstream left open. It stays
open only when it is the last block, since a client must see each block stop before the next
one starts.
"""
upstream_left_text_open: Final = any(
block.get("type") == "text" for block in anthropic_sse_unfinished_blocks(source_chunks).values()
)
rewritten_blocks: Final = anthropic_sse_started_blocks(rewritten_chunks)
if not upstream_left_text_open or not rewritten_blocks:
return None
last_index: Final = max(rewritten_blocks)
return last_index if rewritten_blocks[last_index].get("type") == "text" else None
def _is_empty_text(block: object) -> bool:
fields: Final = as_str_mapping(block)
return fields is not None and fields.get("type") == "text" and not fields.get("text")
def _without_empty_text(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
return {**response, "content": [block for block in response.get("content") or () if not _is_empty_text(block)]}
def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[str]) -> bool:
event: Final = chunk.decode()
event_type: Final = anthropic_sse_event_type(event)
if event_type in _MESSAGE_END_EVENT_TYPES:
return event_type in end_events_sent
return block_left_open is None or anthropic_sse_stopped_block_index(event) != block_left_open
def _sse_chunks_of_events(events: Sequence[str]) -> tuple[bytes, ...]:
return tuple(f"{event}\n\n".encode() for event in events)
def anthropic_sse_chunks_from_response(
assembled: ModelResponse, source_chunks: Sequence[object] = (), *, keep_explanations: bool = False
) -> tuple[bytes, ...]:
@ -110,7 +165,26 @@ def anthropic_sse_chunks_from_response(
nothing the OpenAI round trip made up, with a verdict kept only for a tool call delivered with
the input the classifier judged. Their free-text ``explanation`` was written about the text the
guardrail rewrote, so it is dropped unless the caller rewrites no text (``keep_explanations``).
A source stream that ended in an upstream ``error`` event ends with that event again, never
with a ``message_delta`` or ``message_stop`` the upstream did not send, with its cut-off text
block left open unless another block follows it, and with no finished text block when it read
no text, since Claude Code does not retry a stream that shows a finished block. A response that
cannot be re-emitted, such as one with a tool input that cannot be parsed, leaves a failed stream
its error alone.
"""
error_events: Final = anthropic_sse_error_events(source_chunks)
try:
return _rebuilt_sse_chunks(assembled, source_chunks, error_events, keep_explanations)
except Exception:
if not error_events:
raise
verbose_proxy_logger.warning("Failed Anthropic stream could not be rebuilt, sending its upstream error alone")
return _sse_chunks_of_events(error_events)
def _rebuilt_sse_chunks(
assembled: ModelResponse, source_chunks: Sequence[object], error_events: Sequence[str], keep_explanations: bool
) -> tuple[bytes, ...]:
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
@ -120,7 +194,7 @@ def anthropic_sse_chunks_from_response(
adapter: Final = LiteLLMAnthropicMessagesAdapter()
anthropic_response: Final = adapter.translate_openai_response_to_anthropic(response=assembled)
source_response: Final = assemble_anthropic_sse_stream(source_chunks)
source_response: Final = assemble_anthropic_sse_stream(source_chunks, through_upstream_error=True)
source_content: Final = (
adapter.translate_openai_response_to_anthropic(response=source_response).get("content")
if source_response is not None
@ -129,11 +203,15 @@ def anthropic_sse_chunks_from_response(
unchanged_tool_use_ids: Final = _unchanged_tool_use_ids(
source_content or (), anthropic_response.get("content") or ()
)
return tuple(
FakeAnthropicMessagesStreamIterator(
response=anthropic_response,
message_delta_fields=_rewritten_message_delta_fields(
source_chunks, unchanged_tool_use_ids, keep_explanations
),
).chunks
rewritten_chunks: Final = FakeAnthropicMessagesStreamIterator(
response=_without_empty_text(anthropic_response) if error_events else anthropic_response,
message_delta_fields=_rewritten_message_delta_fields(source_chunks, unchanged_tool_use_ids, keep_explanations),
).chunks
if not error_events:
return tuple(rewritten_chunks)
block_left_open: Final = _text_block_left_open(rewritten_chunks, source_chunks)
end_events_sent: Final = _MESSAGE_END_EVENT_TYPES & anthropic_sse_event_types(source_chunks)
return (
*(chunk for chunk in rewritten_chunks if _sent_by_the_failed_upstream(chunk, block_left_open, end_events_sent)),
*_sse_chunks_of_events(error_events),
)

2
litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
View file

@ -29,6 +29,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
has_tool_with_name,
)
from litellm.litellm_core_utils.provider_affinity import get_caller_session_id
from litellm.litellm_core_utils.routing_common_utils import request_user_api_key_auth
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType]
httpxSpecialProvider,
@ -40,7 +41,6 @@ from litellm.proxy.guardrails.guardrail_hooks.content_text import (
merge_rewritten_text_parts,
)
from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_PROVIDER
from litellm.router_utils.common_utils import request_user_api_key_auth
from litellm.secret_managers.main import get_secret_str
from litellm.types.guardrails import GuardrailEventHooks, Mode
from litellm.types.integrations.custom_logger import (

50
litellm/proxy/guardrails/guardrail_hooks/presidio.py
View file

@ -48,6 +48,7 @@ from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
from litellm.proxy.guardrails.anthropic_sse import (
anthropic_sse_chunks_from_response,
assemble_anthropic_sse_stream,
has_unreadable_json_event,
is_anthropic_sse_stream,
model_response_text,
)
@ -117,7 +118,7 @@ _MAX_FIRST_SSE_FRAME_BYTES: Final = 64 * 1024
@dataclass(frozen=True, slots=True)
class _SsePreface:
"""Complete leading SSE frames with no ``data:`` line, relayed verbatim before the stream shape is decided."""
"""Complete leading SSE frames that carry no data, relayed verbatim before the stream shape is decided."""
raw: bytes
@ -125,17 +126,27 @@ class _SsePreface:
_SSE_FRAME_END: Final = re.compile(rb"\r\n\r\n|\n\n|\r\r")
def _carries_data(frame: bytes) -> bool:
return any(line.startswith(b"data:") and line[len(b"data:") :].strip() for line in frame.splitlines())
def _split_sse_preface(complete_frames: bytes) -> tuple[bytes, bytes]:
"""Split complete frames into ``(frames before the first data-bearing frame, that frame and everything after)``."""
start = 0
for end in _SSE_FRAME_END.finditer(complete_frames):
frame = complete_frames[start : end.end()]
if any(line.startswith(b"data:") for line in frame.splitlines()):
if _carries_data(frame):
return complete_frames[:start], complete_frames[start:]
start = end.end()
return complete_frames, b""
def _holds_json_object_frame(frames: bytes) -> bool:
return any(
line.startswith(b"data:") and line[len(b"data:") :].lstrip().startswith(b"{") for line in frames.splitlines()
)
def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[ModelResponseStream]:
"""Buffered chunks flushed unmasked when a mixed stream shape makes reconstruction impossible."""
if not all_chunks:
@ -150,12 +161,13 @@ def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[
async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGenerator[object, None]:
"""
Relay leading data-less SSE frames (comment keepalives, events without a
``data:`` line) as they complete, and join raw ``bytes`` chunks until they
hold one complete SSE event with a data line, so the stream shape is
decided on a whole frame rather than a transport fragment. Everything
after that first frame is forwarded untouched. The byte cap can only be
reached by a single unterminated frame.
Relay leading data-less SSE frames (comment keepalives, events with no
data) as they complete, and join raw ``bytes`` chunks until they
hold one complete SSE event with a JSON object on its data line, so the
stream shape is decided on a whole frame rather than a transport fragment
or a line that says nothing about the shape. Everything after that frame
is forwarded untouched. The byte cap bounds a single unterminated frame
and a run of frames with no JSON object.
"""
pending = b""
try:
@ -169,7 +181,7 @@ async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGener
if preface:
yield _SsePreface(preface)
pending = classifiable + tail
if classifiable or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
if _holds_json_object_frame(classifiable) or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
break
else:
if pending:
@ -1523,7 +1535,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
first_frame_is_anthropic = (
not passthrough_due_to_unknown_stream_shape
and not all_chunks
and is_anthropic_sse_stream((chunk,))
and self._is_anthropic_first_frame(chunk)
)
if not first_frame_is_anthropic:
passthrough_due_to_unknown_stream_shape = (
@ -1565,13 +1577,29 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
for chunk in all_chunks:
yield chunk
def _is_anthropic_first_frame(self, chunk: bytes) -> bool:
if is_anthropic_sse_stream((chunk,)):
return True
if has_unreadable_json_event((chunk,)):
raise self._unreadable_stream_refusal()
return False
def _unreadable_stream_refusal(self) -> GuardrailRaisedException:
return GuardrailRaisedException(
guardrail_name=self.guardrail_name,
message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured",
should_wrap_with_default_message=False,
)
async def _mask_anthropic_sse_stream(
self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict
) -> tuple[object, ...]:
rest_chunks: Final = [chunk async for chunk in rest]
chunks: Final = (first_chunk, *rest_chunks)
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True)
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True, through_upstream_error=True)
if assembled is None:
if has_unreadable_json_event(chunks):
raise self._unreadable_stream_refusal()
verbose_proxy_logger.warning(
"Presidio apply_to_output: raw SSE stream could not be assembled into a response. "
"Output PII masking was skipped for this response."

3
litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py
View file

@ -14,6 +14,7 @@ from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD
from litellm.router_api import RoutingContext
if TYPE_CHECKING:
from semantic_router.routers import SemanticRouter
@ -134,7 +135,7 @@ class SemanticGuardRouteLoader:
)
encoder: Final = LiteLLMRouterEncoder(
litellm_router_instance=litellm_router,
ctx=RoutingContext(litellm_router),
model_name=embedding_model,
score_threshold=global_threshold,
)

34
litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
View file

@ -25,8 +25,10 @@ from litellm.proxy.common_utils.callback_utils import (
)
from litellm.proxy.guardrails.anthropic_sse import (
anthropic_sse_chunks_from_response,
anthropic_sse_error_events,
assemble_anthropic_sse_stream,
is_raw_sse_stream,
is_sse_error_stream,
)
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
@ -333,7 +335,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
parsed_arguments = arguments
else:
return None, "arguments must be a JSON object"
except (json.JSONDecodeError, TypeError) as exc:
except (json.JSONDecodeError, TypeError, RecursionError) as exc:
verbose_proxy_logger.warning(
"Tool Permission Guardrail: Failed to decode arguments for tool %s: %s",
tool_call.function.name,
@ -957,9 +959,11 @@ class ToolPermissionGuardrail(CustomGuardrail):
yield chunk
return
anthropic_response: Final = assemble_anthropic_sse_stream(all_chunks)
anthropic_response: Final = assemble_anthropic_sse_stream(
all_chunks, restore_identity=bool(anthropic_sse_error_events(all_chunks)), through_upstream_error=True
)
if anthropic_response is None:
if is_raw_sse_stream(all_chunks):
if is_raw_sse_stream(all_chunks) and not is_sse_error_stream(all_chunks):
raise GuardrailRaisedException(
guardrail_name=self.guardrail_name,
message=(
@ -971,16 +975,24 @@ class ToolPermissionGuardrail(CustomGuardrail):
yield chunk
return
anthropic_denials: Final = self._check_assembled_stream(anthropic_response)
if not anthropic_denials:
for chunk in all_chunks:
yield chunk
return
self._modify_response_with_permission_errors(anthropic_response, anthropic_denials)
for sse_chunk in anthropic_sse_chunks_from_response(anthropic_response, all_chunks, keep_explanations=True):
for sse_chunk in self._anthropic_stream_to_deliver(anthropic_response, all_chunks):
yield sse_chunk
def _anthropic_stream_to_deliver(
self, anthropic_response: ModelResponse, source_chunks: Sequence[object]
) -> Sequence[object]:
"""The frames the client gets: on a failed upstream only what the check read, never the upstream's own.
Assembly reads past an upstream error, and a client can read a frame of a broken stream that assembly
does not, so forwarding those frames would pass a tool call nobody checked.
"""
denials: Final = self._check_assembled_stream(anthropic_response)
if denials:
self._modify_response_with_permission_errors(anthropic_response, denials)
elif not anthropic_sse_error_events(source_chunks):
return source_chunks
return anthropic_sse_chunks_from_response(anthropic_response, source_chunks, keep_explanations=True)
def _check_assembled_stream(
self, assembled: ModelResponse
) -> tuple[tuple[ChatCompletionMessageToolCall, PermissionError], ...]:

Some files were not shown because too many files have changed in this diff Show more