Merge upstream main into fix-embedding-pre-call-token-count

This commit is contained in:
m.torres 2026-11-03 15:13:36 +01:00
commit b6030f7e62
No known key found for this signature in database
GPG key ID: 46647ED3C647683E
22 changed files with 1895 additions and 114 deletions

View file

@ -11,7 +11,7 @@ from typing import (
cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict
)
from typing_extensions import TypedDict
from typing_extensions import ReadOnly, TypedDict
from litellm.types.llms.openai import ResponsesAPIResponse
@ -20,15 +20,15 @@ if TYPE_CHECKING:
class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
model: str
messages: list
optional_params: dict
litellm_params: dict
headers: dict
model_response: "ModelResponse"
logging_obj: "LiteLLMLoggingObj"
custom_llm_provider: str
encoding: object
model: ReadOnly[str]
messages: ReadOnly[list]
optional_params: ReadOnly[dict[str, object]]
litellm_params: ReadOnly[dict[str, object]]
headers: ReadOnly[dict]
model_response: ReadOnly["ModelResponse"]
logging_obj: ReadOnly["LiteLLMLoggingObj"]
custom_llm_provider: ReadOnly[str]
encoding: ReadOnly[object]
def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
@ -184,6 +184,7 @@ class ResponsesToCompletionBridgeHandler:
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
logging_obj.record_bridged_chat_request(optional_params, litellm_params)
request_data: Final = self.transformation_handler.transform_request(
model=model,
@ -285,6 +286,7 @@ class ResponsesToCompletionBridgeHandler:
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
logging_obj.record_bridged_chat_request(optional_params, litellm_params)
try:
request_data: Final = self.transformation_handler.transform_request(

View file

@ -1235,10 +1235,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
api_model: Final = model
from litellm.types.utils import CallTypes
setattr(litellm_logging_obj, "call_type", CallTypes.responses.value)
sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params)
request_data: Final = {

View file

@ -9,6 +9,7 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and
"""
import bisect
import copy
import itertools
import os
@ -21,12 +22,16 @@ from urllib.parse import urlparse
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.integrations.prompt_management_base import PromptManagementClient
from litellm.litellm_core_utils.prompt_templates.common_utils import (
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT,
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_as_cache_control,
with_prompt_cache_breakpoint,
without_prompt_cache_breakpoints,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
tool_call_ids_answered_by_client,
@ -327,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object:
return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part))
def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object:
if not _carries_prompt_cache_breakpoint(part):
return part
return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params)
def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]:
if not isinstance(value, (list, tuple)):
return ()
@ -551,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool:
return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS)
def _carries_cache_control(block: object) -> bool:
return _attribute_or_key(block, "cache_control") is not None
def _attribute_or_key(value: object, key: str) -> object | None:
if hasattr(value, key):
return _ATTRIBUTE_VALUE.validate_python(getattr(value, key))
@ -630,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b
return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig)
def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool:
return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages(
model, custom_llm_provider
)
def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None:
provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP)
return provider if isinstance(provider, str) else None
@ -673,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
def _carries_prompt_cache_breakpoint(block: object) -> bool:
return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None
def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
return message.get("role") != "assistant"
def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
return message.get("role") == "tool" and any(
_carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or ()
)
def _part_without_prompt_cache_breakpoint(part: object) -> object:
if not _carries_prompt_cache_breakpoint(part):
return part
return {
key: value
for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items()
if key != "prompt_cache_breakpoint"
}
def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues:
if not _tool_result_carries_prompt_cache_breakpoint(message):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
return cast( # cast-ok: the same tool message with its part marks removed
AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]}
)
def _created_cache_control(original: object, translated: object) -> bool:
return (
_attribute_or_key(original, "cache_control") is None
and _attribute_or_key(translated, "cache_control") is not None
)
def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]:
if translated is original:
return ()
original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or []
translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or []
return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts))
def _part_without_cache_control(part: object) -> object:
return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"}
def _message_dropping_created_cache_controls_between(
message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int
) -> AllMessageValues:
if not any(created):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content")) or []
ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before)
return cast( # cast-ok: the same message with its middle translated marks removed
AllMessageValues,
{
**message,
"content": [
_part_without_cache_control(part)
if is_created and first_dropped <= ordinal < first_kept_after
else part
for part, is_created, ordinal in zip(parts, created, ordinals)
],
},
)
# Set by a caller whose message list is not the one that goes upstream -- today the
# Responses API layer, whose `instructions` only becomes a system message further down.
# Tells this hook to hand role-targeted points to the pass holding the final messages
@ -762,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement):
if not injection_points:
return model, messages, non_default_params
# Create a deep copy of messages to avoid modifying the original list
processed_messages = copy.deepcopy(messages)
message_points: Final = tuple(
cast(CacheControlMessageInjectionPoint, point)
for point in injection_points
@ -775,6 +862,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
request_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base hook's bare dict
custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0])
stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
model_info: Final = routed_model_info(request_params.get("model_info"))
openai_dialect: Final = (
stamped_dialect
if isinstance(stamped_dialect, bool)
@ -783,9 +871,14 @@ class AnthropicCacheControlHook(CustomPromptManagement):
custom_llm_provider,
request_params.get("api_base") or request_params.get("base_url"),
request_params.get("prompt_cache_options"),
routed_model_info(request_params.get("model_info")),
model_info,
)
)
processed_messages = copy.deepcopy(
[without_prompt_cache_breakpoints(message) for message in messages]
if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider)
else messages
)
# A provisional message list defers every role-targeted point to the pass holding
# the final one: a role with no message here may have one there, and settling all
# of them in one pass is what lets config order decide the shared breakpoint
@ -876,7 +969,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
)
if hosted_flag is not None:
return hosted_flag
return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider)
if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info):
return False
if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai":
@ -929,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return int(wire_cache_control is not None) + tool_blocks + envelope_blocks
@staticmethod
def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]:
return tuple(
tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool)
)
@staticmethod
def count_chat_external_cache_breakpoints(optional_params: object) -> int:
params: Final = _validated_object_mapping(optional_params) or {}
return AnthropicCacheControlHook.count_external_cache_breakpoints(
_as_object_list(params.get("tools")), params.get("cache_control")
)
@staticmethod
def count_external_cache_breakpoints_on_messages_route(
tools: Iterable[object] | None, cache_control: object, request_kwargs: object
@ -1084,6 +1190,16 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return message_count + content_count + tool_call_count
@staticmethod
def count_converse_cache_points(message: AllMessageValues) -> int:
content: Final = _as_object_list(_attribute_or_key(message, "content"))
tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or []
marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block))
marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call))
if message.get("role") == "tool":
return int(_carries_cache_control(message) or marked_parts > 0)
return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls
@staticmethod
def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool:
"""Return True if the message already carries any cache_control."""
@ -1142,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement):
return renamed
return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
@staticmethod
def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues:
"""Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's
``cache_control``. Marks with no OpenAI slot are left for the caller to strip."""
if not _may_carry_prompt_cache_breakpoint(message):
return message
parts: Final = _as_object_list(_attribute_or_key(message, "content"))
if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts):
return message
return cast( # cast-ok: the same message with its part marks renamed
AllMessageValues,
{**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]},
)
@staticmethod
def prompt_cache_breakpoints_dropped_from_tool_results(
messages: Sequence[AllMessageValues], drop_params: bool, model: str
) -> tuple[AllMessageValues, ...]:
if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages):
return tuple(messages)
if not drop_params:
raise UnsupportedParamsError(
message=(
f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool "
f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
),
status_code=400,
)
verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model)
return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages)
@staticmethod
def prompt_cache_breakpoints_as_capped_cache_control(
messages: Sequence[AllMessageValues],
drop_params: bool,
sent_breakpoints: Callable[[AllMessageValues], int],
external_breakpoints: Callable[[], int],
) -> tuple[AllMessageValues, ...]:
"""``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's
breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends.
Under ``drop_params`` the first and last translated marks fill the slots the client's own
``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest
turns stay cached, and without it a request over the cap is rejected."""
translated: Final = tuple(
AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params)
for message in messages
)
if all(after is before for before, after in zip(messages, translated)):
return translated
created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated))
created_marks: Final = sum(sum(flags) for flags in created)
if created_marks == 0:
return translated
external: Final = external_breakpoints()
breakpoints: Final = sum(map(sent_breakpoints, translated)) + external
excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS
if excess <= 0:
return translated
if not drop_params:
raise UnsupportedParamsError(
message=(
f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this "
f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} "
f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them."
),
status_code=400,
)
verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params")
marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0))
def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]:
first_dropped: Final = (kept_marks + 1) // 2
first_kept_after: Final = created_marks - kept_marks // 2
return tuple(
_message_dropping_created_cache_controls_between(
message, flags, before, first_dropped, first_kept_after
)
for message, flags, before in zip(translated, created, marks_before)
)
def over_cap(kept_marks: int) -> bool:
return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS
first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap)
return capped_request(max(0, first_over_cap - 1))
@staticmethod
def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
if not _may_carry_prompt_cache_breakpoint(message):
@ -1487,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points"))
if configured:
tools_keeping_marks: Final = tuple(
tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool)
)
tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools)
stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook(
configured,
AnthropicCacheControlHook.count_external_cache_breakpoints(
@ -1782,9 +1982,17 @@ def messages_within_cache_control_cap(
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
return messages_within_reserved_cache_control_cap(
messages,
AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
),
)
def messages_within_reserved_cache_control_cap(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:

View file

@ -19,7 +19,7 @@ from datetime import datetime as dt_object
from enum import Enum
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, TypeVar, cast
from httpx import Response, Timeout
from openai import AsyncStream as OpenAIAsyncStream
@ -99,6 +99,7 @@ from litellm.litellm_core_utils.hidden_params import (
get_hidden_params,
served_from_cache,
set_hidden_param,
set_hidden_params,
)
from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
@ -124,6 +125,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import (
diagnose_zero_cost,
zero_cost_warning,
)
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler
from litellm.litellm_core_utils.logging_utils import (
truncate_base64_in_messages,
truncate_base64_in_messages_async,
@ -144,7 +146,7 @@ from litellm.litellm_core_utils.served_output_texts import (
from litellm.litellm_core_utils.thread_pool_executor import executor
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.containers.main import ContainerObject
from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse
@ -302,6 +304,11 @@ class _EnterpriseAlertingLoggers(NamedTuple):
smtp_email: type[CustomLogger]
class BridgedChatRequest(NamedTuple):
optional_params: Mapping[str, object]
litellm_params: Mapping[str, object]
@functools.cache
def _enterprise_alerting_loggers() -> _EnterpriseAlertingLoggers:
try:
@ -582,6 +589,7 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
_REDACTED_RESULT: Final = TypeAdapter(object)
_LoggedResponse: Final = TypeVar("_LoggedResponse")
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
@ -875,6 +883,7 @@ class Logging(LiteLLMLoggingBaseClass):
# once that response is priced, the same way a non-streamed response is logged
self.client_facing_stream_model: str | None = None
self.client_facing_response_id: str | None = None
self.bridged_chat_request: BridgedChatRequest | None = None
self.zero_cost_warned: bool = False
self.llm_caching_handler: LLMCachingHandler | None = None
@ -3235,6 +3244,40 @@ class Logging(LiteLLMLoggingBaseClass):
return response
return response.model_copy(update={"model": model})
def _as_served_chat_response(self, response: _LoggedResponse) -> _LoggedResponse | ModelResponse:
request: Final = self.bridged_chat_request
if request is None or not isinstance(response, ResponsesAPIResponse):
return response
return self._bridged_chat_response(response, request)
def _as_served_response(
self, response: _LoggedResponse
) -> _LoggedResponse | ModelResponse | TextCompletionResponse:
request: Final = self.bridged_chat_request
if request is None or not isinstance(response, ResponsesAPIResponse):
return response
served: Final = self._bridged_chat_response(response, request)
if self.call_type not in (CallTypes.text_completion.value, CallTypes.atext_completion.value): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
return served
return self._as_text_completion_priced_as_chat(served)
def _as_text_completion_priced_as_chat(self, served: ModelResponse) -> TextCompletionResponse:
text_completion: Final = LiteLLMResponseObjectHandler.convert_chat_to_text_completion(
served, TextCompletionResponse()
)
set_hidden_params(text_completion, dict(get_hidden_params(served) or {}))
return text_completion
def _bridged_chat_response(self, response: ResponsesAPIResponse, request: BridgedChatRequest) -> ModelResponse:
translated: Final = self._translate_responses_api_response_to_model_response(response, request)
served: Final = translated.model_copy(
update={
"id": ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response.id)
}
)
set_hidden_params(served, {**(get_hidden_params(translated) or {}), **(get_hidden_params(response) or {})})
return served
def _with_continued_response_id(
self,
response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
@ -3316,6 +3359,8 @@ class Logging(LiteLLMLoggingBaseClass):
)
result = self.normalize_endpoint_logging_result(result)
if self.stream is not True:
result = self._as_served_response(result) # rebind-ok: bridged chat calls log the served response
logging_result: Final = self.normalize_logging_result(result=result)
if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@ -3670,7 +3715,9 @@ class Logging(LiteLLMLoggingBaseClass):
)
self._add_continued_partial_spend(complete_streaming_response)
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(
self._as_served_chat_response(complete_streaming_response)
)
self.model_call_details["complete_streaming_response"] = logged_streaming_response
self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
## STANDARDIZED LOGGING PAYLOAD
@ -4268,7 +4315,9 @@ class Logging(LiteLLMLoggingBaseClass):
await self._prepare_baseline_cache_estimate(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response)
logged_streaming_response: Final = self._client_facing_streaming_response(
self._as_served_chat_response(complete_streaming_response)
)
self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
## STANDARDIZED LOGGING PAYLOAD
@ -5227,6 +5276,13 @@ class Logging(LiteLLMLoggingBaseClass):
row is built from this response rather than re-parsed from the served body."""
self.bridged_model_response = response
def record_bridged_chat_request(
self, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> None:
self.bridged_chat_request = BridgedChatRequest(
MappingProxyType(dict(optional_params)), MappingProxyType(dict(litellm_params))
)
def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None:
self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response))
@ -5298,6 +5354,9 @@ class Logging(LiteLLMLoggingBaseClass):
):
result = result.response
if isinstance(result, ResponsesAPIResponse):
bridged_request: Final = self.bridged_chat_request
if bridged_request is not None:
return self._bridged_chat_response(result, bridged_request)
return self._translate_responses_api_response_to_model_response(result)
raw_body: Final[object] = result
@ -5349,7 +5408,9 @@ class Logging(LiteLLMLoggingBaseClass):
set_response_cost_in_hidden_params(result, reported_cost)
return result
def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse:
def _translate_responses_api_response_to_model_response(
self, result: ResponsesAPIResponse, request: BridgedChatRequest | None = None
) -> ModelResponse:
"""
Convert a Responses API response into a ModelResponse for spend_logs.
@ -5372,8 +5433,8 @@ class Logging(LiteLLMLoggingBaseClass):
logging_obj=self,
request_data={},
messages=[],
optional_params={},
litellm_params={},
optional_params={} if request is None else dict(request.optional_params),
litellm_params={} if request is None else dict(request.litellm_params),
encoding=litellm.encoding,
)
translated.id = served_id or translated.id

View file

@ -21,12 +21,14 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.batch_utils import InMemoryFile
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAudioObject,
ChatCompletionCachedContent,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionPromptCacheBreakpoint,
@ -2183,6 +2185,9 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool:
_MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = (
"set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)"
)
def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
@ -2262,6 +2267,51 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
)
def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT:
"""OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block.
A ``cache_control`` the client already wrote on the block wins, TTL included."""
marker: Final = block.get("prompt_cache_breakpoint")
if marker is None:
return block
unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"}
control: Final = (
None
if block.get("cache_control") is not None
else _cache_control_for_prompt_cache_breakpoint(marker, drop_params)
)
marked: Final = unmarked if control is None else {**unmarked, "cache_control": control}
return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed
def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None:
try:
breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
except ValidationError:
if not drop_params:
raise UnsupportedParamsError(
message=(
f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got "
f"{marker!r}. Fix the marker, or "
f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it."
),
status_code=400,
)
verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
return None
ttl: Final = breakpoint_mark.get("ttl")
if ttl is not None and not drop_params:
raise UnsupportedParamsError(
message=(
f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint "
f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send "
"a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block "
"where the model offers a 1h cache."
),
status_code=400,
)
return ChatCompletionCachedContent(type="ephemeral")
_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"})
_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])

View file

@ -10,7 +10,7 @@
import asyncio
import copy
import inspect
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, Any, Final
@ -107,7 +107,13 @@ def redacted_response_obj_for_custom_logger(response_obj: object, custom_logger:
def _is_redactable_response(response_obj: object) -> bool:
return isinstance(
response_obj, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse)
response_obj,
(
litellm.ModelResponse,
litellm.TextCompletionResponse,
litellm.ResponsesAPIResponse,
litellm.EmbeddingResponse,
),
) or (isinstance(response_obj, dict) and ("choices" in response_obj or "output" in response_obj))
@ -208,6 +214,7 @@ def _redact_choice_content(choice):
choice.message.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.message, "thinking_blocks"):
choice.message.thinking_blocks = None
_drop_field_when_present(choice.message, "reasoning_items")
_redact_provider_specific_fields(choice.message)
_redact_tool_calls(getattr(choice.message, "tool_calls", None))
_redact_function_call(getattr(choice.message, "function_call", None))
@ -218,12 +225,14 @@ def _redact_choice_content(choice):
choice.delta.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "thinking_blocks"):
choice.delta.thinking_blocks = None
_drop_field_when_present(choice.delta, "reasoning_items")
_redact_provider_specific_fields(choice.delta)
_redact_tool_calls(getattr(choice.delta, "tool_calls", None))
_redact_function_call(getattr(choice.delta, "function_call", None))
elif isinstance(choice, litellm.utils.TextChoices):
_redact_field_when_present(choice, "text")
_redact_field_when_present(choice, "reasoning_content")
_drop_field_when_present(choice, "logprobs")
def _as_mapping(value: object) -> Mapping[str, object] | None:
@ -284,6 +293,11 @@ def _redact_field_when_present(item: object, field: str) -> None:
setattr(item, field, REDACTED_BY_LITELLM)
def _drop_field_when_present(item: object, field: str) -> None:
if getattr(item, field, None) is not None:
setattr(item, field, None)
def _redact_responses_api_output(output_items):
"""Helper to redact ResponsesAPIResponse output items."""
for output_item in output_items:
@ -440,32 +454,22 @@ def _redact_tool_calls_dict(message: Mapping[str, object]) -> None:
function_call["arguments"] = REDACTED_BY_LITELLM
def _redact_model_response_dict_choices(choices, redacted_str: str):
def _redact_model_response_dict_choices(choices: Iterable[object], redacted_str: str) -> None:
for choice in choices:
if isinstance(choice, dict):
_redact_provider_specific_fields_dict(choice)
if choice.get("token_ids") is not None:
choice["token_ids"] = None
if "message" in choice and isinstance(choice["message"], dict):
if choice["message"].get("content") is not None:
choice["message"]["content"] = redacted_str
if choice["message"].get("reasoning_content") is not None:
choice["message"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
_redact_tool_calls_dict(choice["message"])
elif "delta" in choice and isinstance(choice["delta"], dict):
if choice["delta"].get("content") is not None:
choice["delta"]["content"] = redacted_str
if choice["delta"].get("reasoning_content") is not None:
choice["delta"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_tool_calls_dict(choice["delta"])
if (
body := next((choice[key] for key in ("message", "delta") if isinstance(choice.get(key), dict)), None)
) is not None:
for field in ("content", "reasoning_content"):
if body.get(field) is not None:
body[field] = redacted_str
for field in ("thinking_blocks", "reasoning_items", "audio"):
if field in body:
body[field] = None
_redact_tool_calls_dict(body)
elif choice.get("text") is not None:
choice["text"] = redacted_str
else:
@ -588,7 +592,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
return {"text": REDACTED_BY_LITELLM}
_result: Final = copy.deepcopy(result)
if isinstance(_result, litellm.ModelResponse):
if isinstance(_result, (litellm.ModelResponse, litellm.TextCompletionResponse)):
if hasattr(_result, "choices") and _result.choices is not None:
for choice in _result.choices:
_redact_choice_content(choice)

View file

@ -2370,6 +2370,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
)
def translates_prompt_cache_breakpoint(self, model: str) -> bool:
return True
def transform_request(
self,
model: str,
@ -2383,19 +2386,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
"""
## VALIDATE REQUEST
"""Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed)."""
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, cache_control_count
from litellm.litellm_core_utils.prompt_templates.factory import (
anthropic_messages_pt,
)
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
translated_messages: Final = (
list(
AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
messages,
drop_params,
cache_control_count,
lambda: AnthropicCacheControlHook.count_chat_external_cache_breakpoints(optional_params),
)
)
if self.translates_prompt_cache_breakpoint(model)
else messages
)
capped_messages: Final = (
list(
self._messages_within_cache_control_cap(
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
translated_messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
)
)
if drop_params
else messages
else translated_messages
)
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):

View file

@ -23,6 +23,11 @@ from litellm.constants import (
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.integrations.anthropic_cache_control_hook import (
MAX_CACHE_CONTROL_BLOCKS,
AnthropicCacheControlHook,
configured_injection_points,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
@ -131,6 +136,7 @@ from ..common_utils import (
bedrock_capability_model,
bedrock_converse_supports_parallel_tool_use_config,
bedrock_model_accepts_cache_points,
bedrock_model_known_to_accept_cache_points,
bedrock_reasoning_effort_disabled,
bedrock_rejects_stop_sequences,
get_anthropic_beta_from_headers,
@ -204,6 +210,11 @@ def _has_native_text_format(output_config: object) -> bool:
return False
def _cache_mark_kept(block: Mapping[str, object], keeps_cache_marks: bool) -> Mapping[str, object]:
cache_control: Final = block.get("cache_control")
return {"cache_control": cache_control} if keeps_cache_marks and cache_control is not None else {}
class AmazonConverseConfig(BaseConfig):
"""
Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html
@ -243,7 +254,7 @@ class AmazonConverseConfig(BaseConfig):
@staticmethod
def _convert_consecutive_user_messages_to_guarded_text(
messages: list[AllMessageValues], optional_params: dict
messages: list[AllMessageValues], optional_params: dict, keeps_cache_marks: bool = False
) -> list[AllMessageValues]:
"""
Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present
@ -281,7 +292,11 @@ class AmazonConverseConfig(BaseConfig):
new_content = []
for item in content:
if isinstance(item, dict) and item.get("type") == "text":
new_item = {"type": "guarded_text", "text": item["text"]}
new_item = {
"type": "guarded_text",
"text": item["text"],
**_cache_mark_kept(_JSON_OBJECT.validate_python(item), keeps_cache_marks),
}
new_content.append(new_item)
else:
new_content.append(item)
@ -289,7 +304,13 @@ class AmazonConverseConfig(BaseConfig):
messages_copy[user_message_index]["content"] = new_content
elif isinstance(content, str):
# If content is a string, convert it to guarded_text
messages_copy[user_message_index]["content"] = [{"type": "guarded_text", "text": content}]
messages_copy[user_message_index]["content"] = [
{
"type": "guarded_text",
"text": content,
**_cache_mark_kept(_JSON_OBJECT.validate_python(user_message), keeps_cache_marks),
}
]
return messages_copy
@ -1595,6 +1616,74 @@ class AmazonConverseConfig(BaseConfig):
return ChatCompletionTextObject(type="text", text=text)
return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control)
def _messages_with_cache_marks(
self,
messages: Sequence[AllMessageValues],
model: str,
deployment_model: str,
drop_params: bool,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> Sequence[AllMessageValues]:
if not bedrock_model_known_to_accept_cache_points(model, deployment_model, model_info):
return messages
markable: Final = (
AnthropicCacheControlHook.prompt_cache_breakpoints_dropped_from_tool_results(messages, drop_params, model)
if "nova" in model.lower()
else messages
)
return AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
markable,
drop_params,
AnthropicCacheControlHook.count_converse_cache_points,
lambda: self._tool_cache_points_on_the_wire(
model, deployment_model, optional_params, litellm_params, model_info
),
)
def _tool_cache_points_on_the_wire(
self,
model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> int:
tools: Final = optional_params.get("tools")
bedrock_tools, _ = self._process_tools_and_beta(
tools if isinstance(tools, list) else [], model, None, {}, litellm_params, deployment_model, model_info
)
injects_tool_config_point: Final = bool(bedrock_tools) and any(
point.get("location") == "tool_config"
for point in configured_injection_points(optional_params.get("cache_control_injection_points"))
)
return sum(1 for tool in bedrock_tools if "cachePoint" in tool) + int(injects_tool_config_point)
def _guarded_text_keeps_cache_marks(
self,
messages: Sequence[AllMessageValues],
system_content_blocks: Sequence[SystemContentBlock],
model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> bool:
if "guardrailConfig" not in optional_params or not bedrock_model_known_to_accept_cache_points(
model, deployment_model, model_info
):
return False
message_cache_points: Final = sum(map(AnthropicCacheControlHook.count_converse_cache_points, messages))
if message_cache_points == 0:
return False
cache_points: Final = (
sum(1 for block in system_content_blocks if "cachePoint" in block)
+ message_cache_points
+ self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params, model_info)
)
return cache_points <= MAX_CACHE_CONTROL_BLOCKS
def _transform_system_message(
self,
messages: list[AllMessageValues],
@ -2152,22 +2241,27 @@ class AmazonConverseConfig(BaseConfig):
return data
@staticmethod
def _claude_messages_within_cache_control_cap(
self,
messages: Sequence[AllMessageValues],
capability_model: str,
deployment_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
model_info: Mapping[str, object] | None,
) -> Sequence[AllMessageValues]:
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
return messages
from litellm.integrations.anthropic_cache_control_hook import (
messages_within_cache_control_cap,
messages_within_reserved_cache_control_cap,
tool_message_with_single_cache_mark,
)
return messages_within_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
return messages_within_reserved_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)),
self._tool_cache_points_on_the_wire(
capability_model, deployment_model, optional_params, litellm_params, model_info
),
)
async def _async_transform_request(
@ -2183,10 +2277,20 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
model_info: Final = routed_model_info(litellm_params.get("model_info"))
json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params)
marked_messages: Final = self._messages_with_cache_marks(
messages,
capability_model,
model,
drops_unsupported_params(litellm_params.get("drop_params")),
json_optional_params,
litellm_params,
model_info,
)
messages, system_content_blocks = self._transform_system_message(
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
marked_messages, capability_model, model, json_optional_params, litellm_params, model_info
)
),
model=capability_model,
@ -2195,8 +2299,19 @@ class AmazonConverseConfig(BaseConfig):
)
# Convert last user message to guarded_text if guardrailConfig is present
guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params)
messages = self._convert_consecutive_user_messages_to_guarded_text(
self._handle_orphaned_tool_blocks(messages, optional_params), optional_params
guard_input,
optional_params,
keeps_cache_marks=self._guarded_text_keeps_cache_marks(
guard_input,
system_content_blocks,
capability_model,
model,
json_optional_params,
litellm_params,
model_info,
),
)
## TRANSFORMATION ##
@ -2276,10 +2391,20 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
model_info: Final = routed_model_info(litellm_params.get("model_info"))
json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params)
marked_messages: Final = self._messages_with_cache_marks(
messages,
capability_model,
model,
drops_unsupported_params(litellm_params.get("drop_params")),
json_optional_params,
litellm_params,
model_info,
)
messages, system_content_blocks = self._transform_system_message(
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
marked_messages, capability_model, model, json_optional_params, litellm_params, model_info
)
),
model=capability_model,
@ -2288,8 +2413,19 @@ class AmazonConverseConfig(BaseConfig):
)
# Convert last user message to guarded_text if guardrailConfig is present
guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params)
messages = self._convert_consecutive_user_messages_to_guarded_text(
self._handle_orphaned_tool_blocks(messages, optional_params), optional_params
guard_input,
optional_params,
keeps_cache_marks=self._guarded_text_keeps_cache_marks(
guard_input,
system_content_blocks,
capability_model,
model,
json_optional_params,
litellm_params,
model_info,
),
)
_data: Final[CommonRequestObject] = self._transform_request_helper(

View file

@ -1559,22 +1559,63 @@ def bedrock_model_accepts_cache_points(
def bedrock_declared_cache_point_support(model: str, model_info: Mapping[str, object] | None = None) -> bool | None:
if _OPENAI_FAMILY_MODEL_RE.search(model):
return False
explicit_marker_support: Final = _explicit_cache_point_support(model, model_info)
if explicit_marker_support is not None:
return explicit_marker_support
return _mapped_prompt_caching_support(model, model_info)
def bedrock_model_known_to_accept_cache_points(
model: str, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None
) -> bool:
"""Like ``bedrock_model_accepts_cache_points``, but only for the Claude and Nova families Bedrock documents
as taking cache checkpoints, unless the entry sets ``supports_prompt_cache_breakpoint``: an OpenAI
``prompt_cache_breakpoint`` reaches every model of a mixed fleet, not only the ones a client marked for
Bedrock caching, and other families price implicit cache reads yet may reject the cachePoint field."""
deployment_declared: Final = (
None if deployment_model is None else _declared_marker_support(deployment_model, model_info)
)
declared: Final = (
deployment_declared if deployment_declared is not None else _declared_marker_support(model, model_info)
)
return declared is True
def _declared_marker_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
if _OPENAI_FAMILY_MODEL_RE.search(model):
return False
explicit_marker_support: Final = _explicit_cache_point_support(model, model_info)
if explicit_marker_support is not None:
return explicit_marker_support
mapped: Final = _mapped_prompt_caching_support(model, model_info)
if not _documents_cache_checkpoints(model):
return None if mapped is None else False
return mapped is not False
def _documents_cache_checkpoints(model: str) -> bool:
return _is_claude(model) or get_bedrock_base_model(model).startswith("amazon.nova")
def _explicit_cache_point_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
deployment_flag: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint")
if deployment_flag is not None:
return deployment_flag is True
catalog_entry: Final = _catalog_entry_reader(model_info)
map_keys: Final = (model, get_bedrock_base_model(model))
registered_keys: Final = tuple(f"bedrock/{route}{model}" for route in ("", "converse/", "converse_like/"))
explicit_marker_support: Final = next(
return next(
(
entry.get("supports_prompt_cache_breakpoint") is True
for key in (*registered_keys, *map_keys)
for key in (*registered_keys, model, get_bedrock_base_model(model))
if (entry := catalog_entry(key)) is not None and entry.get("supports_prompt_cache_breakpoint") is not None
),
None,
)
if explicit_marker_support is not None:
return explicit_marker_support
def _mapped_prompt_caching_support(model: str, model_info: Mapping[str, object] | None) -> bool | None:
catalog_entry: Final = _catalog_entry_reader(model_info)
map_keys: Final = (model, get_bedrock_base_model(model))
if not any(catalog_entry(key) is not None for key in map_keys):
return None
deployment_caching: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching"))

View file

@ -112,6 +112,9 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig):
def custom_llm_provider(self) -> str:
return "opencode_go"
def translates_prompt_cache_breakpoint(self, model: str) -> bool:
return "claude" in model.lower()
def get_complete_url(
self,
api_base: str | None,

View file

@ -55,6 +55,17 @@ def _sorted_pairs(pairs: Iterable[tuple[str, JsonValue]]) -> tuple[tuple[str, Js
return tuple(sorted(pairs, key=lambda pair: pair[0]))
_CACHE_MARKER_KEYS: Final = frozenset(("cache_control", "prompt_cache_breakpoint"))
def _is_cache_marked(block: Mapping[str, object]) -> bool:
match block:
case {"cache_control": {"type": "ephemeral"}}:
return True
case _:
return block.get("prompt_cache_breakpoint") is not None
def _canonical_bytes(value: object) -> bytes:
return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
@ -66,7 +77,7 @@ def _block_unit(
return _canonical_bytes((envelope, block)), message_run_type
block_type: Final = block.get("type")
block_run_type: Final = block_type if isinstance(block_type, str) and block_type in _TOOL_RUN_BLOCK_TYPES else None
stripped: Final = _sorted_pairs(item for item in block.items() if item[0] != "cache_control")
stripped: Final = _sorted_pairs(item for item in block.items() if item[0] not in _CACHE_MARKER_KEYS)
return _canonical_bytes((envelope, stripped)), message_run_type or block_run_type
@ -264,15 +275,9 @@ class PromptCachingCache:
continue
for content_idx, content_block in enumerate(content):
if isinstance(content_block, dict):
cache_control = content_block.get("cache_control")
if (
cache_control is not None
and isinstance(cache_control, dict)
and cache_control.get("type") == "ephemeral"
):
last_cacheable_message_idx = msg_idx
last_cacheable_content_idx = content_idx
if isinstance(content_block, dict) and _is_cache_marked(content_block):
last_cacheable_message_idx = msg_idx
last_cacheable_content_idx = content_idx
# If no cacheable block found, return empty list (no cacheable prefix)
if last_cacheable_message_idx is None:

View file

@ -475,10 +475,6 @@ def row_key(request_id: str) -> str:
return upstream_id_of(request_id.split("_cache_hit", 1)[0])
def caller_sees_upstream_id(endpoint: Endpoint, *, stream: bool) -> bool:
return (endpoint, stream) != ("messages", True)
def usage_of(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
usage: Final = payload.get("usage")
return JSON.validate_python(usage) if isinstance(usage, dict) else {}
@ -733,10 +729,9 @@ def marked_cell(gateway: Gateway, endpoint: Endpoint, *, stream: bool) -> None:
assert_answered(outcome, marker)
assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream)
assert_usage(endpoint, outcome.usage)
row: Final = success_row(name, marker)
row: Final = success_row(name, marker, outcome.response_id)
assert_priced_row(row, GPT)
if caller_sees_upstream_id(endpoint, stream=stream):
assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id)
assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id)
if not stream:
assert math.isclose(float(outcome.headers["x-litellm-response-cost"]), expected_spend(GPT), rel_tol=1e-9), (
outcome.headers
@ -901,7 +896,7 @@ def assert_marked_sdk_cell(
only_received(wire), expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream
)
assert_usage(endpoint, outcome.usage)
assert_priced_row(success_row(name, marker), GPT)
assert_priced_row(success_row(name, marker, outcome.response_id), GPT)
HOSTILE_OPTIONS: Final[dict[str, JsonValue]] = {

View file

@ -1211,6 +1211,6 @@ def test_non_claude_mantle_id_with_a_bare_bedrock_twin_keeps_its_route_and_spend
cost: Final = _mantle_cost(gateway, _LUNA)
assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(cost), response.text
_assert_success_row(
_spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "responses", cost
_spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "acompletion", cost
)
assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_PATH)]

View file

@ -1,4 +1,3 @@
import base64
import json
import uuid
from collections.abc import Callable
@ -547,14 +546,8 @@ def test_azure_gpt_6_bridged_no_cache_function_requests_each_reach_provider_and_
lambda found: len(found) == 2,
seconds=70,
)
by_response_id: Final = {
(decoded := base64.b64decode(string_value(row["request_id"]).removeprefix("resp_")).decode())
.rsplit("response_id:", 1)[1]: (decoded, row)
for row in rows
}
for response_id in response_ids:
decoded, row = by_response_id[response_id]
assert decoded.startswith("litellm:custom_llm_provider:azure;model_id:"), rows
assert sorted(string_value(row["request_id"]) for row in rows) == sorted(response_ids), rows
for row in rows:
assert (
string_value(row["status"]),
string_value(row["cache_hit"]),
@ -601,11 +594,7 @@ def test_azure_gpt_6_bridged_function_requests_without_cache_field_still_hit_cac
seconds=70,
)
priced, cached = rows
priced_request: Final = base64.b64decode(
string_value(priced["request_id"]).removeprefix("resp_")
).decode()
assert priced_request.startswith("litellm:custom_llm_provider:azure;model_id:"), rows
assert priced_request.endswith(";response_id:resp_cached"), rows
assert string_value(priced["request_id"]) == "resp_cached", rows
assert (
priced["status"],
priced["cache_hit"],

View file

@ -1,5 +1,4 @@
import json
from base64 import b64decode
from collections.abc import Callable
from hashlib import sha256
from typing import Final
@ -100,8 +99,7 @@ def test_bridged_chat_completion_above_log_offload_threshold_logs_and_charges_on
call_rows: Final = [row for row in rows if row["litellm_call_id"] == call_id]
assert len(call_rows) == 1, rows
assert float(str(call_rows[0]["spend"])) == pytest.approx(EXPECTED_SPEND), rows
logged_id: Final = b64decode(str(call_rows[0]["request_id"]).removeprefix("resp_")).decode()
assert logged_id.startswith("litellm:") and f";response_id:{upstream_id}" in logged_id, rows
assert call_rows[0]["request_id"] == upstream_id, rows
key_spend: Final = eventually(
lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)),

View file

@ -1,9 +1,13 @@
import asyncio
import copy
import datetime
import json
import os
import queue
import unittest
from collections.abc import Mapping, Sequence
from collections.abc import AsyncIterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast, get_args
from unittest.mock import ANY, MagicMock, Mock, patch
@ -24,15 +28,15 @@ import litellm
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT, ResponsesAPIResponse
from litellm.types.utils import ModelResponse, StandardLoggingPayload, TextCompletionResponse
if TYPE_CHECKING:
from openai.types.responses import ResponseOutputItem
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse
def test_completion_over_responses_maps_anthropic_search_tool_to_openai_tool_search() -> None:
response_body: Final = {
@ -191,6 +195,417 @@ def test_auto_summary_sends_the_level_the_target_accepts(
assert request_body["reasoning"] == {"effort": "low", "summary": expected_summary}
@dataclass(frozen=True, slots=True)
class _LoggedSuccess:
call_type: str
payload: StandardLoggingPayload
response_obj: object
complete_streaming_response: object
class _LoggedCallTypes(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.seen_by_caller: Final = MappingProxyType(
{
"completion": queue.SimpleQueue[_LoggedSuccess](),
"acompletion": queue.SimpleQueue[_LoggedSuccess](),
}
)
@staticmethod
def _logged(kwargs: Mapping[str, object], response_obj: object, streaming_key: str) -> _LoggedSuccess:
return _LoggedSuccess(
call_type=str(kwargs["call_type"]),
payload=cast(StandardLoggingPayload, kwargs["standard_logging_object"]),
response_obj=response_obj,
complete_streaming_response=kwargs.get(streaming_key),
)
def log_success_event(
self,
kwargs: Mapping[str, object],
response_obj: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None:
self.seen_by_caller["completion"].put(self._logged(kwargs, response_obj, "complete_streaming_response"))
async def async_log_success_event(
self,
kwargs: Mapping[str, object],
response_obj: object,
start_time: datetime.datetime,
end_time: datetime.datetime,
) -> None:
self.seen_by_caller["acompletion"].put(
self._logged(kwargs, response_obj, "async_complete_streaming_response")
)
_GET_WEATHER_TOOL: Final = {
"type": "function",
"function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}},
}
_BRIDGED_PROVIDERS: Final = pytest.mark.parametrize(
("model", "url", "provider_kwargs"),
[
("openai/gpt-5.3-codex", "https://api.openai.com/v1/responses", {}),
(
"azure/gpt-6-sol",
"https://example.invalid/openai/responses",
{"api_base": "https://example.invalid", "api_version": "2025-04-01-preview"},
),
("bedrock_mantle/openai.gpt-5.6-terra", _MANTLE_RESPONSES_URL, {"aws_region_name": "us-east-1"}),
],
ids=["openai-responses-only", "azure-tools", "bedrock-mantle-tools"],
)
_PONG_MESSAGE: Final[Mapping[str, object]] = {
"type": "message",
"id": "msg_logged_call_type",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "pong", "annotations": []}],
}
def _searched_responses_body(model: str) -> Mapping[str, object]:
return {
"id": "resp_logged_call_type",
"object": "response",
"created_at": 1789788253,
"status": "completed",
"model": model.rsplit("/", 1)[-1],
"output": [
{
"type": "web_search_call",
"id": "ws_logged_call_type",
"status": "completed",
"action": {"type": "search", "query": "weather"},
},
_PONG_MESSAGE,
],
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
}
def _responses_sse(response_body: Mapping[str, object]) -> str:
events: Final = (
{"type": "response.created", "response": {**response_body, "status": "in_progress", "output": [], "usage": None}},
{
"type": "response.output_text.delta",
"item_id": "msg_logged_call_type",
"output_index": 1,
"content_index": 0,
"delta": "pong",
},
{"type": "response.completed", "response": response_body},
)
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
def _bridged_request(model: str, provider_kwargs: Mapping[str, object], stream: bool) -> Mapping[str, object]:
return {
"model": model,
"messages": [{"role": "user", "content": "ping"}],
"tools": [_GET_WEATHER_TOOL],
"api_key": "test-key",
"cache": {"no-cache": True},
"stream": stream,
**provider_kwargs,
}
async def _served_chunk_ids(caller: str, request: Mapping[str, object]) -> frozenset[str]:
if caller == "acompletion":
async_stream: Final = await litellm.acompletion(**request)
assert isinstance(async_stream, CustomStreamWrapper)
return frozenset([chunk.id async for chunk in async_stream])
sync_stream: Final = litellm.completion(**request)
assert isinstance(sync_stream, CustomStreamWrapper)
return frozenset(chunk.id for chunk in sync_stream)
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@_BRIDGED_PROVIDERS
@pytest.mark.asyncio
async def test_bridged_chat_call_is_logged_with_the_callers_call_type(
monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object]
) -> None:
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = _searched_responses_body(model)
request: Final = _bridged_request(model, provider_kwargs, stream=False)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith=url).mock(return_value=httpx.Response(200, json=response_body))
if caller == "acompletion":
await litellm.acompletion(**request)
else:
litellm.completion(**request)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert (logged.call_type, logged.payload["call_type"]) == (caller, caller)
assert isinstance(logged.response_obj, ModelResponse)
assert logged.response_obj.id == response_body["id"]
assert logged.response_obj.choices[0].message.content == "pong"
assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong"
assert logged.payload["response_cost"] == litellm.completion_cost(
completion_response=ResponsesAPIResponse.model_validate(response_body), model=model
)
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@_BRIDGED_PROVIDERS
@pytest.mark.asyncio
async def test_bridged_chat_stream_is_logged_as_the_chat_completion_it_streamed(
monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object]
) -> None:
monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = _searched_responses_body(model)
request: Final = _bridged_request(model, provider_kwargs, stream=True)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith=url).mock(
return_value=httpx.Response(
200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"}
)
)
served_ids: Final = await _served_chunk_ids(caller, request)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert isinstance(logged.complete_streaming_response, ModelResponse)
assert logged.response_obj is logged.complete_streaming_response
assert served_ids == {logged.complete_streaming_response.id}
assert logged.complete_streaming_response.choices[0].message.content == "pong"
assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong"
assert logged.payload["response_cost"] == litellm.completion_cost(
completion_response=ResponsesAPIResponse.model_validate(response_body), model=model
)
def _json_message(item_id: str, document: Mapping[str, object]) -> Mapping[str, object]:
return {
"type": "message",
"id": item_id,
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": json.dumps(document), "annotations": []}],
}
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@pytest.mark.asyncio
async def test_bridged_structured_output_call_is_logged_with_the_json_document_the_client_got(
monkeypatch: pytest.MonkeyPatch, caller: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = {
**_searched_responses_body("openai/gpt-5.3-codex"),
"output": [_json_message("msg_first", {"answer": "pong"}), _json_message("msg_second", {"answer": "again"})],
}
request: Final = {
**_bridged_request("openai/gpt-5.3-codex", {}, stream=False),
"response_format": {"type": "json_object"},
}
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=response_body)
)
served: Final = await litellm.acompletion(**request) if caller == "acompletion" else litellm.completion(**request)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert isinstance(served, ModelResponse)
assert isinstance(logged.response_obj, ModelResponse)
served_message: Final = served.choices[0].message
logged_message: Final = logged.response_obj.choices[0].message
assert json.loads(served_message.content) == {"answer": "pong"}
assert logged_message.content == served_message.content
assert logged_message.provider_specific_fields == served_message.provider_specific_fields
assert logged.payload["response"]["choices"][0]["message"]["content"] == served_message.content
async def _logged_text_completion(
monkeypatch: pytest.MonkeyPatch, caller: str
) -> tuple[TextCompletionResponse, _LoggedSuccess, Mapping[str, object]]:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = _searched_responses_body("openai/gpt-5.3-codex")
request: Final = {"model": "openai/gpt-5.3-codex", "prompt": "ping", "api_key": "test-key"}
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=response_body)
)
served: Final = (
await litellm.atext_completion(**request) if caller == "acompletion" else litellm.text_completion(**request)
)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert isinstance(served, TextCompletionResponse)
return served, logged, response_body
@pytest.mark.parametrize(("caller", "call_type"), [("completion", "text_completion"), ("acompletion", "atext_completion")])
@pytest.mark.asyncio
async def test_bridged_text_completion_is_logged_and_priced_under_the_callers_call_type(
monkeypatch: pytest.MonkeyPatch, caller: str, call_type: str
) -> None:
served, logged, response_body = await _logged_text_completion(monkeypatch, caller)
assert (logged.call_type, logged.payload["call_type"]) == (call_type, call_type)
assert getattr(logged.response_obj, "id") == served.id == response_body["id"]
assert logged.payload["response_cost"] == litellm.completion_cost(
completion_response=ResponsesAPIResponse.model_validate(response_body), model="openai/gpt-5.3-codex"
)
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@pytest.mark.asyncio
async def test_bridged_text_completion_is_logged_as_the_text_completion_it_served(
monkeypatch: pytest.MonkeyPatch, caller: str
) -> None:
served, logged, _ = await _logged_text_completion(monkeypatch, caller)
assert isinstance(logged.response_obj, TextCompletionResponse)
assert logged.response_obj.choices[0].text == served.choices[0].text == "pong"
assert logged.payload["response"]["object"] == served.object == "text_completion"
assert logged.payload["response"]["choices"][0]["text"] == "pong"
@pytest.mark.asyncio
async def test_a_bridged_chat_response_waits_on_the_usage_ack_like_any_chat_response(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from tests.unit.proxy.spend_tracking.test_usage_meter import CommitBarrier
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
database: Final = CommitBarrier()
runtime: Final = UsageCaptureRuntime()
runtime.start(database.write, UsageCaptureSettings(batch_size=2))
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True})
logging_obj: Final = Logging(
model="openai/gpt-5.3-codex",
messages=[],
stream=False,
call_type="acompletion",
start_time=datetime.datetime.now(),
litellm_call_id="bridged-chat",
function_id="bridged-chat",
)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=_searched_responses_body("openai/gpt-5.3-codex"))
)
served: Final = await litellm.acompletion(
model="openai/gpt-5.3-codex",
messages=[{"role": "user", "content": "ping"}],
api_key="test-key",
litellm_logging_obj=logging_obj,
)
response: Final = asyncio.create_task(capture_proxy_response(logging_obj, served))
try:
await database.entered.wait()
assert not response.done()
assert database.committed == ()
database.release.set()
await response
assert "bridged-chat:inbound" in database.committed
finally:
database.release.set()
await runtime.stop(1)
await response
def _served_message_start_id(events: Sequence[bytes]) -> object:
lines: Final = b"".join(events).decode().splitlines()
starts: Final = tuple(
json.loads(line.removeprefix("data:")) for line in lines if line.startswith("data:") and '"message_start"' in line
)
assert len(starts) == 1
return starts[0]["message"]["id"]
@pytest.mark.parametrize("stream", [False, True], ids=["non-stream", "stream"])
@pytest.mark.asyncio
async def test_bridged_messages_call_is_logged_as_anthropic_messages_under_the_id_the_client_got(
monkeypatch: pytest.MonkeyPatch, stream: bool
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": [_PONG_MESSAGE]}
upstream: Final = (
httpx.Response(200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"})
if stream
else httpx.Response(200, json=response_body)
)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(return_value=upstream)
served: Final = await litellm.anthropic.messages.acreate(
model="openai/gpt-5.3-codex",
max_tokens=64,
messages=[{"role": "user", "content": "ping"}],
api_key="test-key",
stream=stream,
)
served_id: Final = (
_served_message_start_id([event async for event in cast(AsyncIterator[bytes], served)])
if stream
else cast(Mapping[str, object], served)["id"]
)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller["acompletion"].get, timeout=10)
assert (logged.call_type, logged.payload["call_type"]) == ("anthropic_messages", "anthropic_messages")
assert logged.payload["id"] == served_id
assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong"
@pytest.mark.parametrize("caller", ["completion", "acompletion"])
@pytest.mark.asyncio
async def test_bridged_chat_stream_keeps_the_upstream_headers_when_its_output_cannot_be_translated(
monkeypatch: pytest.MonkeyPatch, caller: str
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
recorder: Final = _LoggedCallTypes()
monkeypatch.setattr(litellm, "callbacks", [recorder])
response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": []}
request: Final = _bridged_request("openai/gpt-5.3-codex", {}, stream=True)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(
200,
text=_responses_sse(response_body),
headers={"content-type": "text/event-stream", "x-request-id": "req_upstream"},
)
)
await _served_chunk_ids(caller, request)
logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10)
assert isinstance(logged.complete_streaming_response, ModelResponse)
assert logged.complete_streaming_response.id == response_body["id"]
assert logged.payload["hidden_params"]["additional_headers"]["llm_provider-x-request-id"] == "req_upstream"
def test_convert_chat_completion_messages_to_responses_api_image_input():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,

View file

@ -1401,6 +1401,109 @@ def _marks_the_route_sends(model: str, custom_llm_provider: str | None, messages
_SECOND_SEARCH_CALL: Final = {**_SEARCH_CALL, "id": "srvtoolu_02Search"}
_CLIENT_MARKED_CONVERSATION: Final = (
{"role": "system", "content": [{"type": "text", "text": "Handbook", "prompt_cache_breakpoint": {"mode": "explicit"}}]},
{"role": "user", "content": [{"type": "text", "text": "First", "prompt_cache_breakpoint": {"mode": "explicit"}}]},
{"role": "assistant", "content": "First answer"},
{"role": "user", "content": "Second"},
)
def _hook_result(
model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object
) -> tuple[list[AllMessageValues], Mapping[str, object]]:
_, processed, returned_params = AnthropicCacheControlHook().get_chat_completion_prompt(
model=model,
messages=cast(list[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_CONVERSATION))),
non_default_params={"cache_control_injection_points": list(points), **params},
prompt_id=None,
prompt_variables=None,
dynamic_callback_params={},
)
return processed, returned_params
def _hook_output(
model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object
) -> list[AllMessageValues]:
return _hook_result(model, points, **params)[0]
def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_claude():
request: Final = AnthropicConfig().transform_request(
model="claude-sonnet-5-5",
messages=_hook_output("anthropic/claude-sonnet-5-5"),
optional_params={"max_tokens": 64},
litellm_params={},
headers={},
)
assert _marks_in(request["system"]) == 0
assert [_marks_in(message) for message in request["messages"]] == [0, 0, 1]
assert "prompt_cache_breakpoint" not in json.dumps(request)
@pytest.mark.parametrize(
"sibling_model_infos",
[
pytest.param((), id="one-deployment"),
pytest.param(
({"supports_prompt_cache_breakpoint": True, "base_model": "anthropic.claude-unmapped-v1:0"},),
id="beside-a-deployment-of-the-same-profile-with-a-base-model",
),
],
)
@pytest.mark.parametrize("route", ["bedrock", "bedrock/converse"])
def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_a_flagged_bedrock_profile(
local_model_cost_map, sibling_model_infos: tuple[Mapping[str, object], ...], route: str
):
from litellm import Router
model: Final = f"{route}/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged"
Router(
model_list=[
{
"model_name": f"claude-profile-{position}",
"litellm_params": {"model": model, "aws_region_name": "us-east-1"},
"model_info": model_info,
}
for position, model_info in enumerate(({"supports_prompt_cache_breakpoint": True}, *sibling_model_infos))
]
)
processed, params = _hook_result(model)
assert "prompt_cache_breakpoint" not in json.dumps(processed)
assert "prompt_cache_options" not in params
assert [_marks_in(message) for message in processed] == [0, 0, 0, 1]
def test_tool_config_injection_point_leaves_client_prompt_cache_breakpoints_on_claude():
processed: Final = _hook_output("anthropic/claude-sonnet-5-5", points=({"location": "tool_config"},))
assert [message["content"] for message in processed[:2]] == [
message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2]
]
@pytest.mark.parametrize(
("model", "params"),
[
pytest.param("openai/gpt-5.6", {"api_base": "https://gateway.example.com/v1"}, id="reads-them"),
pytest.param("openai/gpt-4o", {}, id="openai-model-that-predates-them"),
pytest.param("hosted_vllm/qwen3", {"api_base": "http://vllm.example.com/v1"}, id="self-hosted"),
pytest.param("azure_ai/qa-unflagged-gpt", {"api_base": "https://qa.services.ai.azure.com/models"}, id="foundry"),
],
)
def test_cache_control_injection_points_keep_client_prompt_cache_breakpoints_on_a_route_that_does_not_translate_them(
model: str, params: Mapping[str, object]
):
processed: Final = _hook_output(model, **params)
assert [message["content"] for message in processed[:2]] == [
message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2]
]
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[
@ -4965,6 +5068,33 @@ class TestMalformedInjectionPointsAreIgnored:
assert kwargs == {"cache_control_injection_points": value}
def test_capping_thousands_of_marks_on_one_tool_result_counts_the_request_a_logarithmic_number_of_times():
def marked_parts(prefix: str, count: int) -> list[dict]:
return [
{"type": "text", "text": f"{prefix} {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(count)
]
messages: Final = [
{"role": "user", "content": marked_parts("part", 4)},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "call_1", "content": marked_parts("result", 2000)},
]
sent_breakpoints: Final = MagicMock(side_effect=AnthropicCacheControlHook.count_converse_cache_points)
capped: Final = AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control(
cast(List[AllMessageValues], messages), True, sent_breakpoints, lambda: 0
)
assert [part.get("cache_control") for part in capped[0]["content"]] == [{"type": "ephemeral"}] * 3 + [None]
assert ["cache_control" in part for part in capped[2]["content"]] == [False] * 1997 + [True] * 3
assert sent_breakpoints.call_count < 3 * 20
def test_cache_control_hook_leaves_the_duplicate_tool_result_warning_to_the_request_builder(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
):

View file

@ -112,6 +112,57 @@ def test_redaction_reuses_message_rules_for_request_body_and_provider_fields(sha
assert result == original_result and params == original_params
def _reasoning_items() -> list[dict[str, object]]:
return [{"type": "reasoning", "id": "rs_redaction", "summary": [{"type": "summary_text", "text": SECRET}]}]
def _message_response() -> ModelResponse:
return ModelResponse(
choices=[{"index": 0, "message": {"role": "assistant", "content": "answer", "reasoning_items": _reasoning_items()}}]
)
def _delta_choices() -> dict[str, object]:
return {"choices": [{"index": 0, "delta": {"content": "answer", "reasoning_items": _reasoning_items()}}]}
@pytest.mark.parametrize(
("details", "result"),
[
pytest.param({}, _message_response(), id="message_object"),
pytest.param({}, _message_response().model_dump(), id="message_dict"),
pytest.param({}, _delta_choices(), id="delta_dict"),
pytest.param(
{"stream": True, "complete_streaming_response": ModelResponseStream(**_delta_choices())},
None,
id="stream_chunk",
),
pytest.param({"standard_logging_object": {"response": _message_response().model_dump()}}, None, id="logged"),
pytest.param({"standard_logging_object": {"response": _delta_choices()}}, None, id="logged_delta"),
],
)
def test_redaction_hides_reasoning_item_summaries(details: dict[str, object], result: object) -> None:
assert SECRET in str(details) + str(result)
redacted: Final = perform_redaction(details, result)
assert SECRET not in str(redacted) and SECRET not in str(details), (redacted, details)
def test_an_opted_out_callback_gets_a_text_completion_with_its_answer_redacted() -> None:
logprobs: Final = {"tokens": [SECRET], "token_logprobs": [-0.1], "text_offset": [0], "top_logprobs": [{SECRET: -0.1}]}
response: Final = TextCompletionResponse(
choices=[{"index": 0, "text": SECRET, "logprobs": logprobs, "finish_reason": "stop"}]
)
redacted: Final = redacted_response_obj_for_custom_logger(response, CustomLogger(message_logging=False))
assert isinstance(redacted, TextCompletionResponse)
assert redacted.choices[0].text == "redacted-by-litellm"
assert SECRET not in redacted.model_dump_json()
assert response.choices[0].text == SECRET, "the caller's own response is not redacted"
def test_redaction_redacts_reasoning_copied_into_hidden_params_in_place() -> None:
hidden_params: Final = {"model_id": "deployment", "provider_specific_fields": _provider_fields()}
original_hidden_params: Final = copy.deepcopy(hidden_params)

View file

@ -1,4 +1,5 @@
import copy
import itertools
import json
import logging
from collections.abc import Iterator
@ -4422,6 +4423,184 @@ def test_transform_request_includes_cache_control():
assert result["cache_control"] == {"type": "ephemeral"}
def _anthropic_body(messages: list, litellm_params: dict | None = None, tools: list | None = None) -> dict:
return AnthropicConfig().transform_request(
model="claude-sonnet-5-5",
messages=messages,
optional_params={"max_tokens": 100, **({"tools": tools} if tools else {})},
litellm_params=litellm_params or {},
headers={},
)
def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control():
body: Final = _anthropic_body(
[
{
"role": "system",
"content": [
{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}
],
},
{
"role": "user",
"content": [
{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}},
{
"type": "text",
"text": "Bye",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
"prompt_cache_breakpoint": {"mode": "explicit"},
},
],
},
]
)
assert body["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]
assert body["messages"] == [
{
"role": "user",
"content": [
{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}},
{"type": "text", "text": "Bye", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
],
}
]
@pytest.mark.parametrize(
"marker, expected_block",
[
({"mode": "explicit", "ttl": "30m"}, {"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}),
({"mode": "implicit"}, {"type": "text", "text": "Hello"}),
],
)
def test_transform_request_prompt_cache_breakpoint_without_claude_equivalent(marker, expected_block):
messages: Final = [{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]}]
with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
_anthropic_body(copy.deepcopy(messages))
body: Final = _anthropic_body(messages, litellm_params={"drop_params": True})
assert body["messages"] == [{"role": "user", "content": [expected_block]}]
def test_transform_request_names_the_marker_ttl_rule_for_a_ttl_only_claude_takes():
messages: Final = [
{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}]}
]
with pytest.raises(litellm.utils.UnsupportedParamsError, match="optional ttl of '30m' or '12h'"):
_anthropic_body(messages)
def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap():
client_marked_part: Final = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}}
marked_parts: Final = [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(1, 5)
]
messages: Final = [{"role": "user", "content": [client_marked_part, *marked_parts]}]
tools: Final = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}]
with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"):
_anthropic_body(copy.deepcopy(messages), tools=copy.deepcopy(tools))
body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools)
assert [part.get("cache_control") for part in body["messages"][0]["content"]] == [
{"type": "ephemeral"},
{"type": "ephemeral"},
None,
None,
{"type": "ephemeral"},
]
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap():
config: Final = AnthropicConfig()
optional_params: Final = config.map_openai_params(
non_default_params={
"max_tokens": 100,
"tools": [
{
"type": "tool_search_tool_regex_20251119",
"name": "tool_search_tool_regex",
"cache_control": {"type": "ephemeral"},
}
],
},
optional_params={},
model="claude-sonnet-5-5",
drop_params=False,
)
marked_turns: Final = [
{
"role": "user",
"content": [{"type": "text", "text": f"turn {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}],
}
for index in range(4)
]
def body(litellm_params: dict) -> dict:
return config.transform_request(
model="claude-sonnet-5-5",
messages=copy.deepcopy(marked_turns),
optional_params=copy.deepcopy(optional_params),
litellm_params=litellm_params,
headers={},
)
with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"):
body({})
capped: Final = body({"drop_params": True})
assert capped["tools"][0]["cache_control"] == {"type": "ephemeral"}
sent_parts: Final = itertools.chain.from_iterable(message["content"] for message in capped["messages"])
assert [part.get("cache_control") for part in sent_parts] == [
{"type": "ephemeral"},
{"type": "ephemeral"},
None,
{"type": "ephemeral"},
]
@pytest.mark.parametrize(
"unsent_marks",
[
pytest.param(
[
{
"role": "assistant",
"content": [{"type": "text", "text": "Noted.", "prompt_cache_breakpoint": {"mode": "explicit"}}],
}
],
id="assistant_part",
),
pytest.param(
[{"role": "user", "content": "Also this.", "prompt_cache_breakpoint": {"mode": "explicit"}}],
id="message_level",
),
],
)
def test_transform_request_caps_only_the_marks_claude_receives(unsent_marks: list[dict]):
marked_parts: Final = [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} for index in range(4)
]
body: Final = _anthropic_body(
[{"role": "user", "content": marked_parts}, *unsent_marks, {"role": "user", "content": "Go on."}]
)
sent_marks: Final = [
part.get("cache_control")
for part in itertools.chain.from_iterable(message["content"] for message in body["messages"])
if isinstance(part, dict)
]
assert sent_marks.count({"type": "ephemeral"}) == 4
def test_compaction_block_empty_list_not_added():
"""
Test that empty compaction_blocks list is not added to provider_specific_fields.

View file

@ -10073,6 +10073,461 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control):
assert "cachePoint" not in json.dumps(transformed_messages)
def test_converse_carries_prompt_cache_breakpoint_to_cache_point():
request: Final = litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
{
"role": "system",
"content": [
{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}
],
},
{
"role": "user",
"content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}],
},
],
optional_params={},
litellm_params={},
)
assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}]
assert request["messages"] == [
{"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]}
]
def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap():
messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(5)
],
}
]
def transform(litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=copy.deepcopy(messages),
optional_params={},
litellm_params=litellm_params,
)
with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"):
transform({})
cache_point: Final = {"cachePoint": {"type": "default"}}
assert transform({"drop_params": True})["messages"] == [
{
"role": "user",
"content": [
{"text": "part 0"},
cache_point,
{"text": "part 1"},
cache_point,
{"text": "part 2"},
{"text": "part 3"},
cache_point,
{"text": "part 4"},
cache_point,
],
}
]
_CONVERSE_CAP_TOOL: Final[dict[str, object]] = {
"type": "function",
"function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}},
}
@pytest.mark.parametrize(
"tool_cache_params",
[
{"tools": [{**_CONVERSE_CAP_TOOL, "cache_control": {"type": "ephemeral"}}]},
{
"tools": [_CONVERSE_CAP_TOOL],
"cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}],
},
],
ids=["client_tool_cache_control", "tool_config_injection_point"],
)
def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params: dict[str, object]):
messages: Final = [
{
"role": "user",
"content": [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(4)
],
}
]
def transform(litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=copy.deepcopy(messages),
optional_params=copy.deepcopy(tool_cache_params),
litellm_params=litellm_params,
)
with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"):
transform({})
request: Final = transform({"drop_params": True})
cache_point: Final = {"cachePoint": {"type": "default"}}
assert request["messages"][0]["content"] == [
{"text": "part 0"},
cache_point,
{"text": "part 1"},
cache_point,
{"text": "part 2"},
{"text": "part 3"},
cache_point,
]
assert request["toolConfig"]["tools"][-1] == cache_point
@pytest.mark.parametrize(
"unsent_tool_params",
[
{
"tools": [
_CONVERSE_CAP_TOOL,
{"type": "tool_search_tool_regex_20251119", "name": "find", "cache_control": {"type": "ephemeral"}},
]
},
{"tools": [_CONVERSE_CAP_TOOL, {"type": "web_search", "cache_control": {"type": "ephemeral"}}]},
{
"tools": [{"type": "web_search"}],
"cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}],
},
],
ids=["tool_search_tool", "built_in_tool", "tool_config_point_with_no_tool_sent"],
)
@pytest.mark.parametrize("litellm_params", [{}, {"drop_params": True}], ids=["strict", "drop_params"])
def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent(
unsent_tool_params: dict[str, object], litellm_params: dict[str, object]
):
request: Final = litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(4)
],
}
],
optional_params=copy.deepcopy(unsent_tool_params),
litellm_params=dict(litellm_params),
)
cache_point: Final = {"cachePoint": {"type": "default"}}
assert request["messages"][0]["content"] == [
{"text": "part 0"},
cache_point,
{"text": "part 1"},
cache_point,
{"text": "part 2"},
cache_point,
{"text": "part 3"},
cache_point,
]
assert cache_point not in request.get("toolConfig", {}).get("tools", [])
def _converse_tool_turn(*result_parts: dict[str, object]) -> list[dict[str, object]]:
return [
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
},
{"role": "tool", "tool_call_id": "call_1", "content": list(result_parts)},
]
def _marked_text_parts(count: int) -> list[dict[str, object]]:
return [
{"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(count)
]
def _marked_tool_result_parts() -> list[dict[str, object]]:
return [
{"type": "text", "text": f"result {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}
for index in range(2)
]
def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks():
request: Final = litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
{"role": "user", "content": _marked_text_parts(3)},
*_converse_tool_turn(*_marked_tool_result_parts()),
],
optional_params={"tools": [_CONVERSE_CAP_TOOL]},
litellm_params={},
)
assert len(_collect_cache_points(request["messages"])) == 4
def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap():
messages: Final = [
{"role": "user", "content": _marked_text_parts(4)},
*_converse_tool_turn(*_marked_tool_result_parts()),
]
def transform(litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=copy.deepcopy(messages),
optional_params={"tools": [_CONVERSE_CAP_TOOL]},
litellm_params=litellm_params,
)
with pytest.raises(litellm.UnsupportedParamsError, match="this request has 5"):
transform({})
request: Final = transform({"drop_params": True})
cache_point: Final = {"cachePoint": {"type": "default"}}
assert request["messages"][0]["content"] == [
{"text": "part 0"},
cache_point,
{"text": "part 1"},
cache_point,
{"text": "part 2"},
cache_point,
{"text": "part 3"},
]
assert len(_collect_cache_points(request["messages"][1:])) == 1
def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result():
messages: Final = [
{
"role": "system",
"content": [{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}],
},
{"role": "user", "content": "Look it up."},
*_converse_tool_turn(*_marked_tool_result_parts()),
]
def transform(litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="us.amazon.nova-lite-v1:0",
messages=copy.deepcopy(messages),
optional_params={"tools": [_CONVERSE_CAP_TOOL]},
litellm_params=litellm_params,
)
with pytest.raises(litellm.UnsupportedParamsError, match="tool result"):
transform({})
request: Final = transform({"drop_params": True})
assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}]
assert _collect_cache_points(request["messages"]) == []
def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_when_flagged(local_model_cost_map):
from litellm import Router
flagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged"
unflagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/mistral"
Router(
model_list=[
{
"model_name": "claude-profile",
"litellm_params": {"model": f"bedrock/{flagged_arn}", "aws_region_name": "us-east-1"},
"model_info": {"supports_prompt_cache_breakpoint": True},
},
{
"model_name": "mistral-profile",
"litellm_params": {"model": f"bedrock/{unflagged_arn}", "aws_region_name": "us-east-1"},
},
]
)
def user_content(model: str, ttl: dict) -> list:
return litellm.AmazonConverseConfig()._transform_request(
model=model,
messages=[
{
"role": "user",
"content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}],
}
],
optional_params={},
litellm_params={},
)["messages"][0]["content"]
assert user_content(flagged_arn, {}) == [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]
assert user_content(unflagged_arn, {}) == [{"text": "Hello"}]
assert user_content(unflagged_arn, {"ttl": "30m"}) == [{"text": "Hello"}]
def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_model(local_model_cost_map):
from litellm import Router
opted_in_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-next"
opted_out_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-opted-out"
Router(
model_list=[
{
"model_name": "opted-in",
"litellm_params": {"model": f"bedrock/{opted_in_arn}", "aws_region_name": "us-east-1"},
"model_info": {"supports_prompt_cache_breakpoint": True},
},
{
"model_name": "opted-out",
"litellm_params": {"model": f"bedrock/{opted_out_arn}", "aws_region_name": "us-east-1"},
"model_info": {"supports_prompt_cache_breakpoint": False},
},
]
)
def user_content(model: str, base_model: str, ttl: dict) -> list:
return litellm.AmazonConverseConfig()._transform_request(
model=model,
messages=[
{
"role": "user",
"content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}],
}
],
optional_params={},
litellm_params={"base_model": base_model},
)["messages"][0]["content"]
assert user_content(opted_in_arn, "anthropic.claude-unmapped-v1:0", {}) == [
{"text": "Hello"},
{"cachePoint": {"type": "default"}},
]
assert user_content(opted_out_arn, "anthropic.claude-sonnet-5-5", {"ttl": "30m"}) == [{"text": "Hello"}]
def test_converse_translates_prompt_cache_breakpoint_only_for_families_that_take_cache_points(local_model_cost_map):
litellm.register_model(
{"zai.glm-next-v1:0": {"litellm_provider": "bedrock_converse", "mode": "chat", "supports_prompt_caching": True}}
)
def user_content(model: str, mark: dict) -> list:
return litellm.AmazonConverseConfig()._transform_request(
model=model,
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", **mark}]}],
optional_params={},
litellm_params={},
)["messages"][0]["content"]
marker: Final = {"prompt_cache_breakpoint": {"mode": "explicit"}}
client_mark: Final = {"cache_control": {"type": "ephemeral"}}
cache_point: Final = {"cachePoint": {"type": "default"}}
assert user_content("zai.glm-next-v1:0", marker) == [{"text": "Hello"}]
assert user_content("zai.glm-next-v1:0", client_mark) == [{"text": "Hello"}, cache_point]
assert user_content("us.anthropic.claude-next-v1:0", marker) == [{"text": "Hello"}, cache_point]
assert user_content("us.amazon.nova-next-v1:0", marker) == [{"text": "Hello"}, cache_point]
def test_converse_prompt_cache_breakpoint_on_guarded_user_text_keeps_its_cache_point(local_model_cost_map):
guardrail: Final = {"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}}
def request(system_marks: int) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
*(
{
"role": "system",
"content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}],
}
for i in range(system_marks)
),
{
"role": "user",
"content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}],
},
],
optional_params=dict(guardrail),
litellm_params={},
)
assert request(0)["messages"][0]["content"] == [
{"guardContent": {"text": {"text": "Hello"}}},
{"cachePoint": {"type": "default"}},
]
with pytest.raises(litellm.UnsupportedParamsError):
request(4)
client_marked: Final = litellm.AmazonConverseConfig()._transform_request(
model="anthropic.claude-sonnet-5-5",
messages=[
*(
{
"role": "system",
"content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}],
}
for i in range(4)
),
{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]},
],
optional_params=dict(guardrail),
litellm_params={},
)
assert sum("cachePoint" in block for block in client_marked["system"]) == 4
assert client_marked["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}]
def test_converse_guarded_user_text_keeps_no_cache_point_on_a_model_not_known_to_take_one(local_model_cost_map):
request: Final = litellm.AmazonConverseConfig()._transform_request(
model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/unmapped1",
messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}],
optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}},
litellm_params={},
)
assert request["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}]
def _converse_request_with_30m_breakpoint(model: str, litellm_params: dict) -> dict:
return litellm.AmazonConverseConfig()._transform_request(
model=model,
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}
],
}
],
optional_params={},
litellm_params=litellm_params,
)
def test_converse_rejects_30m_breakpoint_ttl_only_where_a_cache_point_is_written():
claude: Final = "anthropic.claude-sonnet-5-5"
with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
_converse_request_with_30m_breakpoint(claude, {})
assert _converse_request_with_30m_breakpoint(claude, {"drop_params": True})["messages"] == [
{"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]}
]
assert _converse_request_with_30m_breakpoint("meta.llama3-1-8b-instruct-v1:0", {})["messages"] == [
{"role": "user", "content": [{"text": "Hello"}]}
]
@pytest.mark.parametrize(
"model, expected_supports_tool_call",
[

View file

@ -14,6 +14,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.opencode.chat.transformation import OpenCodeGoMessagesChatConfig
from litellm.llms.opencode.common_utils import go_headers
@ -591,3 +592,22 @@ def test_stream_chunk_builder_reads_list_cost_from_dict_chunks():
assert assembled is not None
assert assembled._hidden_params["response_cost"] == GO_REPORTED_USAGE["opencode"]["list_cost"]
@pytest.mark.parametrize(("model", "cache_marks"), [("qwen3.8-flash", 0), ("minimax-m3", 0), ("claude-haiku-5-5", 4)])
def test_go_messages_route_carries_prompt_cache_breakpoint_to_cache_control_only_for_claude(model: str, cache_marks: int):
marked_notes: Final = [
{"type": "text", "text": f"Note {n}", "prompt_cache_breakpoint": {"mode": "explicit"}} for n in range(4)
]
request: Final = OpenCodeGoMessagesChatConfig().transform_request(
model=model,
messages=[{"role": "user", "content": [*marked_notes, {"type": "text", "text": "Answer in one line."}]}],
optional_params={"max_tokens": 64},
litellm_params={},
headers={},
)
wire: Final = json.dumps(request)
assert wire.count('"cache_control"') == cache_marks
assert "prompt_cache_breakpoint" not in wire

View file

@ -1,4 +1,5 @@
import asyncio
from collections.abc import Mapping
from typing import Final, cast
import pytest
@ -377,6 +378,32 @@ def test_extract_cacheable_prefix_mixed_string_and_list_content():
assert isinstance(result[2]["content"], list)
def test_prompt_cache_breakpoint_marks_the_same_cacheable_prefix_as_cache_control():
def _messages(marker: Mapping[str, Mapping[str, str]]) -> list[AllMessageValues]:
return cast( # cast-ok: the text part type does not declare the prompt_cache_breakpoint key
list[AllMessageValues],
[
{
"role": "system",
"content": [
{"type": "text", "text": "Here is the full text of a complex legal agreement" * 400, **marker}
],
},
{"role": "user", "content": "what are the key terms?"},
],
)
marked_key: Final = PromptCachingCache.get_prompt_caching_cache_key(
_messages({"prompt_cache_breakpoint": {"mode": "explicit"}}), None
)
assert marked_key is not None
assert marked_key == PromptCachingCache.get_prompt_caching_cache_key(
_messages({"cache_control": {"type": "ephemeral"}}), None
)
assert PromptCachingCache.get_prompt_caching_cache_key(_messages({}), None) is None
@pytest.mark.asyncio
async def test_same_cacheable_prefix_routes_to_same_deployment() -> None:
from litellm import Router