mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-13 03:59:12 +00:00
fix(anthropic): carry cache_control to prompt_cache_breakpoint on the /v1/messages to Responses bridge (#46082)
A caller hitting /v1/messages with a cache_control mark and an OpenAI gpt-5.6 or newer deployment behind it lost the mark when the request was rebuilt as a Responses request, so OpenAI never cached at the boundary the caller chose. The bridge now rewrites each marked system, user and tool_result block to prompt_cache_breakpoint on the same block when the deployment reads breakpoints, keeps the system prompt as a developer message so its mark survives, and still strips the mark for deployments that do not read it The hook's gate becomes public so the bridge and the hook share one answer for which deployments read breakpoints, and the one-block swap lives next to the existing breakpoint helper in prompt_templates
This commit is contained in:
ryan-crabbe-berri
GitHub
parent
ca4169d20a
commit
c2d0726a08
9 changed files with 328 additions and 39 deletions
|
|
@ -326,7 +326,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
openai_dialect: Final = (
|
||||
stamped_dialect
|
||||
if isinstance(stamped_dialect, bool)
|
||||
else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
|
||||
model,
|
||||
non_default_params.get("custom_llm_provider"),
|
||||
non_default_params.get("api_base") or non_default_params.get("base_url"),
|
||||
|
|
@ -384,7 +384,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return model, processed_messages, non_default_params
|
||||
|
||||
@staticmethod
|
||||
def _targets_openai_prompt_cache_breakpoint(
|
||||
def targets_openai_prompt_cache_breakpoint(
|
||||
model: str | None,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: object = None,
|
||||
|
|
@ -786,7 +786,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return AnthropicCacheControlHook._stamped(
|
||||
points,
|
||||
"_litellm_openai_dialect",
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
|
||||
model, custom_llm_provider, api_base, prompt_cache_options
|
||||
),
|
||||
)
|
||||
|
|
@ -1103,7 +1103,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
if not injection_points:
|
||||
return messages, system
|
||||
|
||||
openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
|
||||
model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
|
||||
)
|
||||
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ from litellm.types.llms.openai import (
|
|||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionUserMessage,
|
||||
PromptCacheBreakpoint,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
|
|
@ -1793,6 +1794,16 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
|
|||
return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key
|
||||
|
||||
|
||||
def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
|
||||
"""Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block."""
|
||||
if block.get("cache_control") is None:
|
||||
return block
|
||||
unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"}
|
||||
return cast( # cast-ok: same block shape as the input with the marker key renamed
|
||||
_MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit"))
|
||||
)
|
||||
|
||||
|
||||
LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"})
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,14 @@ Handler for the Anthropic v1/messages -> OpenAI Responses API path.
|
|||
Used when the target model is an OpenAI or Azure model.
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator, Coroutine, Mapping
|
||||
from collections.abc import AsyncIterator, Coroutine, Mapping, Sequence
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
|
||||
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
|
||||
from litellm.types.llms.anthropic import (
|
||||
AllAnthropicMessageValues,
|
||||
|
|
@ -27,6 +31,7 @@ from .streaming_iterator import AnthropicResponsesStreamWrapper
|
|||
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
|
||||
|
||||
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
|
||||
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
|
||||
|
||||
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
|
||||
|
||||
|
|
@ -45,6 +50,24 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec
|
|||
return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
|
||||
|
||||
|
||||
def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool:
|
||||
provider: Final = forwarded_kwargs.get("custom_llm_provider")
|
||||
return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
|
||||
model,
|
||||
provider if isinstance(provider, str) else None,
|
||||
forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"),
|
||||
forwarded_kwargs.get("prompt_cache_options"),
|
||||
)
|
||||
|
||||
|
||||
def _blocks_with_prompt_cache_breakpoints(content: object) -> Sequence[Mapping[str, object]]:
|
||||
"""The content's blocks with each ``cache_control`` mark rewritten, empty when it is not a list of blocks."""
|
||||
try:
|
||||
return [cache_control_as_prompt_cache_breakpoint(block) for block in _BLOCK_LIST.validate_python(content)]
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def _build_responses_kwargs(
|
||||
*,
|
||||
max_tokens: int,
|
||||
|
|
@ -68,10 +91,28 @@ def _build_responses_kwargs(
|
|||
"""
|
||||
Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
|
||||
"""
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
|
||||
reads_prompt_cache_breakpoint: Final = _reads_prompt_cache_breakpoint(model, forwarded_kwargs)
|
||||
request_messages: Final[AnthropicRequestMessages] = (
|
||||
[
|
||||
{
|
||||
**message,
|
||||
"content": list(_blocks_with_prompt_cache_breakpoints(message.get("content")))
|
||||
or message.get("content"),
|
||||
}
|
||||
for message in messages
|
||||
]
|
||||
if reads_prompt_cache_breakpoint
|
||||
else messages
|
||||
)
|
||||
request_system: Final = (
|
||||
(list(_blocks_with_prompt_cache_breakpoints(system)) or system) if reads_prompt_cache_breakpoint else system
|
||||
)
|
||||
|
||||
# Build a typed AnthropicMessagesRequest for the adapter
|
||||
request_data: Final[AnthropicMessagesRequest] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"messages": request_messages,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if context_management:
|
||||
|
|
@ -80,8 +121,8 @@ def _build_responses_kwargs(
|
|||
request_data["output_config"] = output_config
|
||||
if metadata:
|
||||
request_data["metadata"] = metadata
|
||||
if system:
|
||||
request_data["system"] = system
|
||||
if request_system:
|
||||
request_data["system"] = request_system
|
||||
if temperature is not None:
|
||||
request_data["temperature"] = temperature
|
||||
if thinking:
|
||||
|
|
@ -96,7 +137,6 @@ def _build_responses_kwargs(
|
|||
request_data["output_format"] = output_format
|
||||
|
||||
anthropic_request: Final = AnthropicMessagesRequest(**request_data)
|
||||
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
|
||||
responses_kwargs: Final = _ADAPTER.translate_request(
|
||||
anthropic_request,
|
||||
include_encrypted_reasoning=_provider_returns_encrypted_reasoning(
|
||||
|
|
|
|||
|
|
@ -131,12 +131,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
def _tool_result_output_value(
|
||||
output_text: str,
|
||||
file_parts: tuple[dict[str, str], ...], # mutable-ok: json content parts
|
||||
tool_result: Mapping[str, object],
|
||||
) -> str | list[dict[str, str]]: # mutable-ok: API message payload
|
||||
"""Plain string output, or a part list when document file parts are present."""
|
||||
if not file_parts:
|
||||
return output_text
|
||||
"""Plain string output, or a part list when document file parts or a cache breakpoint are present.
|
||||
|
||||
OpenAI reads a breakpoint only on a part, so a marked result ends on a part that carries it.
|
||||
"""
|
||||
prompt_cache_breakpoint: Final = tool_result.get("prompt_cache_breakpoint")
|
||||
text_parts: Final = [{"type": "input_text", "text": output_text}] if output_text else []
|
||||
return [*text_parts, *file_parts]
|
||||
parts: Final = [*text_parts, *file_parts]
|
||||
if not parts or (not file_parts and prompt_cache_breakpoint is None):
|
||||
return output_text
|
||||
return [*parts[:-1], with_prompt_cache_breakpoint(parts[-1], prompt_cache_breakpoint)]
|
||||
|
||||
@staticmethod
|
||||
def _translate_midturn_system_content_to_responses(
|
||||
|
|
@ -327,7 +333,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": tool_use_id,
|
||||
"output": self._tool_result_output_value(output_text, tool_file_parts),
|
||||
"output": self._tool_result_output_value(output_text, tool_file_parts, block),
|
||||
}
|
||||
)
|
||||
if tool_image_parts:
|
||||
|
|
|
|||
|
|
@ -367,6 +367,142 @@ def test_anthropic_sdk_system_string_gets_the_injected_marker(bridge: _Bridge) -
|
|||
bridge.spend.landed(bridge.injecting_off, raw.headers["x-litellm-call-id"], None)
|
||||
|
||||
|
||||
_EPHEMERAL: Final[anthropic.types.CacheControlEphemeralParam] = {"type": "ephemeral"}
|
||||
_OPTED_IN: Final[Mapping[str, JsonValue]] = {"prompt_cache_options": {"mode": "explicit"}}
|
||||
|
||||
|
||||
def _anthropic_message(
|
||||
client: anthropic.Anthropic,
|
||||
model: str,
|
||||
messages: Sequence[anthropic.types.MessageParam],
|
||||
*,
|
||||
system: Sequence[anthropic.types.TextBlockParam] = (),
|
||||
stream: bool = False,
|
||||
) -> anthropic.types.Message:
|
||||
if not stream:
|
||||
return client.messages.create(
|
||||
model=model, max_tokens=64, system=system, messages=messages, extra_body=dict(pcb.NO_CACHE)
|
||||
)
|
||||
with client.messages.stream(
|
||||
model=model, max_tokens=64, system=system, messages=messages, extra_body=dict(pcb.NO_CACHE)
|
||||
) as events:
|
||||
return events.get_final_message()
|
||||
|
||||
|
||||
def _marked_system() -> list[anthropic.types.TextBlockParam]:
|
||||
return [
|
||||
{"type": "text", "text": "Answer briefly"},
|
||||
{"type": "text", "text": "Stable rules", "cache_control": _EPHEMERAL},
|
||||
]
|
||||
|
||||
|
||||
def _marked_user(marker: str) -> anthropic.types.MessageParam:
|
||||
return {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": pcb.prompt(marker), "cache_control": _EPHEMERAL},
|
||||
{"type": "text", "text": "and nothing else"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["unary", "stream"])
|
||||
def test_anthropic_sdk_cache_control_reaches_the_responses_wire_as_a_prompt_cache_breakpoint_on_the_same_block(
|
||||
bridge: _Bridge, stream: bool
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with bridge.gateway.scenario() as scenario, _anthropic(bridge) as client:
|
||||
model: Final = scenario.model(model=pcb.MODEL, api_base=bridge.api_base, **_OPTED_IN)
|
||||
message: Final = _anthropic_message(
|
||||
client, model, [_marked_user(marker)], system=_marked_system(), stream=stream
|
||||
)
|
||||
_anthropic_text(message, marker)
|
||||
body: Final = _wire_body(pcb.posted(bridge.wire, marker), stream=stream)
|
||||
assert "instructions" not in body, body
|
||||
assert body["input"] == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Answer briefly"},
|
||||
{"type": "input_text", "text": "Stable rules", "prompt_cache_breakpoint": pcb.EXPLICIT},
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": pcb.prompt(marker), "prompt_cache_breakpoint": pcb.EXPLICIT},
|
||||
{"type": "input_text", "text": "and nothing else"},
|
||||
],
|
||||
},
|
||||
], body
|
||||
|
||||
|
||||
def test_anthropic_sdk_cache_control_sends_no_mark_to_a_deployment_that_does_not_read_breakpoints(
|
||||
bridge: _Bridge,
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with _anthropic(bridge) as client:
|
||||
message: Final = _anthropic_message(client, bridge.off, [_marked_user(marker)], system=_marked_system())
|
||||
_anthropic_text(message, marker)
|
||||
request: Final = pcb.posted(bridge.wire, marker)
|
||||
body: Final = _wire_body(request)
|
||||
assert body["instructions"] == "Answer briefly\nStable rules", body
|
||||
assert body["input"] == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": pcb.prompt(marker)},
|
||||
{"type": "input_text", "text": "and nothing else"},
|
||||
],
|
||||
}
|
||||
], body
|
||||
assert b"cache_control" not in request.body and b"prompt_cache_breakpoint" not in request.body, request.body
|
||||
|
||||
|
||||
def test_anthropic_sdk_cache_control_on_a_tool_result_marks_its_function_call_output(bridge: _Bridge) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
messages: Final[list[anthropic.types.MessageParam]] = [
|
||||
{"role": "user", "content": pcb.prompt(marker)},
|
||||
{"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "lookup", "input": {}}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "call_1", "content": "found it", "cache_control": _EPHEMERAL}
|
||||
],
|
||||
},
|
||||
]
|
||||
with bridge.gateway.scenario() as scenario, _anthropic(bridge) as client:
|
||||
model: Final = scenario.model(model=pcb.MODEL, api_base=bridge.api_base, **_OPTED_IN)
|
||||
message: Final = _anthropic_message(client, model, messages)
|
||||
_anthropic_text(message, marker)
|
||||
request: Final = pcb.posted(bridge.wire, marker)
|
||||
_wire_body(request)
|
||||
assert pcb.function_output(pcb.input_items(request), "call_1") == [
|
||||
{"type": "input_text", "text": "found it", "prompt_cache_breakpoint": pcb.EXPLICIT}
|
||||
], "OpenAI caches up to a breakpoint on a function_call_output part (prompt caching guide, read 2026-10-12)"
|
||||
|
||||
|
||||
def test_anthropic_sdk_cache_control_on_an_assistant_turn_is_dropped(bridge: _Bridge) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
messages: Final[list[anthropic.types.MessageParam]] = [
|
||||
{"role": "user", "content": "first question"},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "earlier answer", "cache_control": _EPHEMERAL}]},
|
||||
{"role": "user", "content": pcb.prompt(marker)},
|
||||
]
|
||||
with bridge.gateway.scenario() as scenario, _anthropic(bridge) as client:
|
||||
model: Final = scenario.model(model=pcb.MODEL, api_base=bridge.api_base, **_OPTED_IN)
|
||||
message: Final = _anthropic_message(client, model, messages)
|
||||
_anthropic_text(message, marker)
|
||||
request: Final = pcb.posted(bridge.wire, marker)
|
||||
_wire_body(request)
|
||||
assert pcb.content_of(pcb.input_items(request), "assistant") == [
|
||||
{"type": "output_text", "text": "earlier answer"}
|
||||
], "OpenAI accepts a breakpoint on output_text but writes no cache for it (live gpt-5.6, 2026-10-12)"
|
||||
|
||||
|
||||
def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)}
|
||||
|
|
|
|||
|
|
@ -3070,7 +3070,7 @@ class TestOpenAIPromptCacheBreakpoint:
|
|||
],
|
||||
)
|
||||
def test_dialect_resolution(self, model, provider, expected):
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(model, provider) is expected
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, provider) is expected
|
||||
|
||||
def test_count_covers_both_marker_kinds(self):
|
||||
message = {
|
||||
|
|
@ -3476,8 +3476,8 @@ class TestChatPathProviderStamp:
|
|||
from unittest.mock import patch
|
||||
|
||||
with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve:
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False
|
||||
resolve.assert_not_called()
|
||||
|
||||
def test_litellm_proxy_target_keeps_anthropic_style_markers(self):
|
||||
|
|
@ -3813,7 +3813,7 @@ class TestHostedOpenAIDialectFlag:
|
|||
def test_flagged_non_openai_deployment_is_eligible(self, monkeypatch):
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle")
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
is True
|
||||
)
|
||||
|
||||
|
|
@ -3821,7 +3821,7 @@ class TestHostedOpenAIDialectFlag:
|
|||
"""The api_base check exists to sniff for api.openai.com, which a Bedrock host never is."""
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle")
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
|
||||
self.MANTLE_MODEL,
|
||||
"bedrock_mantle",
|
||||
api_base="https://bedrock-runtime.us-east-1.amazonaws.com",
|
||||
|
|
@ -3832,7 +3832,7 @@ class TestHostedOpenAIDialectFlag:
|
|||
def test_flag_set_false_keeps_the_deployment_ineligible(self, monkeypatch):
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=False)
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
is False
|
||||
)
|
||||
|
||||
|
|
@ -3840,14 +3840,14 @@ class TestHostedOpenAIDialectFlag:
|
|||
def test_an_odd_typed_flag_keeps_the_deployment_ineligible(self, monkeypatch, flag):
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=flag)
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
is False
|
||||
)
|
||||
|
||||
def test_unflagged_non_openai_deployment_stays_ineligible(self, monkeypatch):
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=None)
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
is False
|
||||
)
|
||||
|
||||
|
|
@ -3855,7 +3855,7 @@ class TestHostedOpenAIDialectFlag:
|
|||
"""A flagged entry does not license a different provider serving the same model string."""
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle")
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "azure") is False
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "azure") is False
|
||||
)
|
||||
|
||||
def test_openai_entries_still_go_through_the_api_base_check(self, monkeypatch):
|
||||
|
|
@ -3863,7 +3863,7 @@ class TestHostedOpenAIDialectFlag:
|
|||
assert litellm.model_cost["gpt-5.6"]["supports_prompt_cache_breakpoint"] is True
|
||||
assert litellm.model_cost["gpt-5.6"]["litellm_provider"] == "openai"
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(
|
||||
"gpt-5.6", "openai", api_base="https://some-compatible-host.example.com"
|
||||
)
|
||||
is False
|
||||
|
|
@ -3871,8 +3871,8 @@ class TestHostedOpenAIDialectFlag:
|
|||
|
||||
def test_azure_hosted_gpt_5_6_remains_ineligible(self):
|
||||
"""Regression guard: the openai entry's flag must not leak to another provider."""
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-5.6", "azure") is False
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("azure/gpt-5.6", None) is False
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("gpt-5.6", "azure") is False
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("azure/gpt-5.6", None) is False
|
||||
|
||||
REGIONAL_MODEL = "bedrock_mantle/us-east-1/openai.gpt-5.6-sol"
|
||||
|
||||
|
|
@ -3880,15 +3880,15 @@ class TestHostedOpenAIDialectFlag:
|
|||
"""``bedrock_mantle/<region>/<model>`` is a documented routing form the map keys without the region."""
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle")
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle")
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle")
|
||||
is True
|
||||
)
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, None) is True
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, None) is True
|
||||
|
||||
def test_region_prefixed_deployment_honors_a_flag_set_false(self, monkeypatch):
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=False)
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle")
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle")
|
||||
is False
|
||||
)
|
||||
|
||||
|
|
@ -3897,31 +3897,31 @@ class TestHostedOpenAIDialectFlag:
|
|||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle")
|
||||
gov_model = "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-sol"
|
||||
self._register(monkeypatch, gov_model, "bedrock_mantle", flag=None)
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(gov_model, "bedrock_mantle") is False
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(gov_model, "bedrock_mantle") is False
|
||||
|
||||
def test_region_free_entry_does_not_license_another_provider(self, monkeypatch):
|
||||
"""The candidate keys are built for the request's provider, so a flagged Mantle row stays Mantle's."""
|
||||
self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle")
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("us-east-1/openai.gpt-5.6-sol", "azure")
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("us-east-1/openai.gpt-5.6-sol", "azure")
|
||||
is False
|
||||
)
|
||||
|
||||
def test_unmapped_deployment_name_stays_ineligible(self):
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("azure/my-deployment", None) is False
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-deployment", "azure") is False
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("azure/my-deployment", None) is False
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("my-deployment", "azure") is False
|
||||
|
||||
def test_bare_name_colliding_with_an_openai_row_reads_its_own_provider_entry(self, monkeypatch):
|
||||
"""The Responses layer hands the hook a bare deployment name plus its provider. The openai row keyed by
|
||||
that bare name neither answers for the deployment nor stops the lookup of the provider's own entry."""
|
||||
self._register(monkeypatch, "gpt-collide", "openai")
|
||||
self._register(monkeypatch, "azure_ai/gpt-collide", "azure_ai")
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is True
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is True
|
||||
|
||||
def test_bare_name_colliding_with_an_openai_row_stays_ineligible_without_its_own_flag(self, monkeypatch):
|
||||
self._register(monkeypatch, "gpt-collide", "openai")
|
||||
self._register(monkeypatch, "azure_ai/gpt-collide", "azure_ai", flag=None)
|
||||
assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is False
|
||||
assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is False
|
||||
|
||||
def test_the_support_check_reads_the_served_providers_row_for_a_provider_stripped_name(self, monkeypatch):
|
||||
"""The chat-to-Responses bridge holds the provider-stripped deployment name and the provider that
|
||||
|
|
@ -3962,7 +3962,7 @@ class TestBedrockMantleGptShipsTheOpenAIDialect:
|
|||
def test_shipped_entry_makes_the_deployment_eligible(self):
|
||||
assert supports_openai_prompt_cache_breakpoint(self.MANTLE_MODEL) is True
|
||||
assert (
|
||||
AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle")
|
||||
is True
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
add_system_prompt_to_messages,
|
||||
cache_control_as_prompt_cache_breakpoint,
|
||||
encrypted_content_from_signature,
|
||||
encrypted_reasoning_signature,
|
||||
get_file_ids_from_messages,
|
||||
|
|
@ -2263,3 +2264,19 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w
|
|||
messages: list[dict[str, object]],
|
||||
) -> None:
|
||||
assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages)
|
||||
|
||||
|
||||
def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None:
|
||||
marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}}
|
||||
|
||||
assert cache_control_as_prompt_cache_breakpoint(marked) == {
|
||||
"type": "text",
|
||||
"text": "rules",
|
||||
"prompt_cache_breakpoint": {"mode": "explicit"},
|
||||
}
|
||||
|
||||
|
||||
def test_a_block_without_cache_control_is_returned_untouched() -> None:
|
||||
unmarked: Final = {"type": "text", "text": "rules"}
|
||||
|
||||
assert cache_control_as_prompt_cache_breakpoint(unmarked) is unmarked
|
||||
|
|
|
|||
|
|
@ -200,3 +200,76 @@ async def test_streaming_hands_the_logging_object_the_message_id_the_caller_is_s
|
|||
message_start = next(e for e in events if e["type"] == "message_start")
|
||||
assert message_start["message"]["id"].startswith("msg_")
|
||||
assert logging_obj.streamed_anthropic_message_id == message_start["message"]["id"]
|
||||
|
||||
|
||||
OPENAI_API: Final = {"custom_llm_provider": "openai", "api_base": "https://api.openai.com/v1"}
|
||||
OTHER_HOST: Final = {"custom_llm_provider": "openai", "api_base": "https://gateway.example/v1"}
|
||||
EPHEMERAL: Final = {"type": "ephemeral"}
|
||||
|
||||
|
||||
def _request_with_client_cache_marks(cache_control: dict[str, str] | None) -> dict[str, object]:
|
||||
mark: Final = {} if cache_control is None else {"cache_control": cache_control}
|
||||
return {
|
||||
"max_tokens": 1024,
|
||||
"system": [{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful.", **mark}],
|
||||
"messages": [
|
||||
{"role": "user", "content": [{"type": "text", "text": "look", **mark}, {"type": "text", "text": "now"}]}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra_kwargs",
|
||||
[
|
||||
pytest.param(OPENAI_API, id="api.openai.com"),
|
||||
pytest.param({**OTHER_HOST, "prompt_cache_options": {"mode": "implicit"}}, id="other-host-opted-in"),
|
||||
],
|
||||
)
|
||||
def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_the_same_block(
|
||||
extra_kwargs: dict[str, object],
|
||||
) -> None:
|
||||
responses_kwargs: Final = _build_responses_kwargs(
|
||||
model="openai/gpt-5.6",
|
||||
extra_kwargs=extra_kwargs,
|
||||
**_request_with_client_cache_marks({**EPHEMERAL, "ttl": "1h"}),
|
||||
)
|
||||
|
||||
assert "instructions" not in responses_kwargs
|
||||
assert responses_kwargs["input"] == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Be concise."},
|
||||
{"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": {"mode": "explicit"}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "look", "prompt_cache_breakpoint": {"mode": "explicit"}},
|
||||
{"type": "input_text", "text": "now"},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "extra_kwargs"),
|
||||
[
|
||||
pytest.param("openai/gpt-5.5", OPENAI_API, id="model-older-than-gpt-5.6"),
|
||||
pytest.param("openai/gpt-5.6", OTHER_HOST, id="other-host-not-opted-in"),
|
||||
],
|
||||
)
|
||||
def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_read_prompt_cache_breakpoint(
|
||||
model: str, extra_kwargs: dict[str, object]
|
||||
) -> None:
|
||||
marked: Final = _build_responses_kwargs(
|
||||
model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(EPHEMERAL)
|
||||
)
|
||||
unmarked: Final = _build_responses_kwargs(
|
||||
model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(None)
|
||||
)
|
||||
|
||||
assert marked == unmarked
|
||||
|
|
|
|||
|
|
@ -2156,7 +2156,7 @@ class TestPromptCacheBreakpointToResponses:
|
|||
}
|
||||
]
|
||||
|
||||
def test_assistant_and_tool_result_blocks_drop_breakpoint(self):
|
||||
def test_assistant_blocks_drop_breakpoint_and_a_tool_result_keeps_it_on_its_last_output_part(self):
|
||||
items = _ADAPTER.translate_messages_to_responses_input(
|
||||
[
|
||||
{"role": "user", "content": [{"type": "text", "text": "q"}]},
|
||||
|
|
@ -2180,8 +2180,14 @@ class TestPromptCacheBreakpointToResponses:
|
|||
},
|
||||
]
|
||||
)
|
||||
assert len(items) == 4
|
||||
assert not _contains_key(items, "prompt_cache_breakpoint")
|
||||
*earlier_items, tool_output = items
|
||||
assert len(earlier_items) == 3
|
||||
assert not _contains_key(earlier_items, "prompt_cache_breakpoint")
|
||||
assert tool_output == {
|
||||
"type": "function_call_output",
|
||||
"call_id": "toolu_01",
|
||||
"output": [{"type": "input_text", "text": "r", "prompt_cache_breakpoint": self.EXPLICIT}],
|
||||
}
|
||||
|
||||
def test_prompt_cache_options_forwarded_to_responses_kwargs(self):
|
||||
from litellm.llms.anthropic.pass_through.responses_adapters.handler import (
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue