diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index cb0a3f0603b..8f542a87a3a 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -326,7 +326,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): openai_dialect: Final = ( stamped_dialect if isinstance(stamped_dialect, bool) - else AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( model, non_default_params.get("custom_llm_provider"), non_default_params.get("api_base") or non_default_params.get("base_url"), @@ -384,7 +384,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): return model, processed_messages, non_default_params @staticmethod - def _targets_openai_prompt_cache_breakpoint( + def targets_openai_prompt_cache_breakpoint( model: str | None, custom_llm_provider: str | None, api_base: object = None, @@ -786,7 +786,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): return AnthropicCacheControlHook._stamped( points, "_litellm_openai_dialect", - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( model, custom_llm_provider, api_base, prompt_cache_options ), ) @@ -1103,7 +1103,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): if not injection_points: return messages, system - openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options") ) breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 0243141eda2..591cae6dfa4 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -29,6 +29,7 @@ from litellm.types.llms.openai import ( ChatCompletionThinkingBlock, ChatCompletionToolParam, ChatCompletionUserMessage, + PromptCacheBreakpoint, ) from litellm.types.utils import ( Choices, @@ -1793,6 +1794,16 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key +def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: + """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" + if block.get("cache_control") is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "cache_control"} + return cast( # cast-ok: same block shape as the input with the marker key renamed + _MarkedT, with_prompt_cache_breakpoint(unmarked, PromptCacheBreakpoint(mode="explicit")) + ) + + LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"}) diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index f20762789dd..5c5fb71c295 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -4,10 +4,14 @@ Handler for the Anthropic v1/messages -> OpenAI Responses API path. Used when the target model is an OpenAI or Azure model. """ -from collections.abc import AsyncIterator, Coroutine, Mapping +from collections.abc import AsyncIterator, Coroutine, Mapping, Sequence from typing import Final, TypeAlias +from pydantic import TypeAdapter, ValidationError + import litellm +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( AllAnthropicMessageValues, @@ -27,6 +31,7 @@ from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]] +_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]]) _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() @@ -45,6 +50,24 @@ def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: objec return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) +def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, object]) -> bool: + provider: Final = forwarded_kwargs.get("custom_llm_provider") + return AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, + provider if isinstance(provider, str) else None, + forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"), + forwarded_kwargs.get("prompt_cache_options"), + ) + + +def _blocks_with_prompt_cache_breakpoints(content: object) -> Sequence[Mapping[str, object]]: + """The content's blocks with each ``cache_control`` mark rewritten, empty when it is not a list of blocks.""" + try: + return [cache_control_as_prompt_cache_breakpoint(block) for block in _BLOCK_LIST.validate_python(content)] + except ValidationError: + return () + + def _build_responses_kwargs( *, max_tokens: int, @@ -68,10 +91,28 @@ def _build_responses_kwargs( """ Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). """ + forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + reads_prompt_cache_breakpoint: Final = _reads_prompt_cache_breakpoint(model, forwarded_kwargs) + request_messages: Final[AnthropicRequestMessages] = ( + [ + { + **message, + "content": list(_blocks_with_prompt_cache_breakpoints(message.get("content"))) + or message.get("content"), + } + for message in messages + ] + if reads_prompt_cache_breakpoint + else messages + ) + request_system: Final = ( + (list(_blocks_with_prompt_cache_breakpoints(system)) or system) if reads_prompt_cache_breakpoint else system + ) + # Build a typed AnthropicMessagesRequest for the adapter request_data: Final[AnthropicMessagesRequest] = { "model": model, - "messages": messages, + "messages": request_messages, "max_tokens": max_tokens, } if context_management: @@ -80,8 +121,8 @@ def _build_responses_kwargs( request_data["output_config"] = output_config if metadata: request_data["metadata"] = metadata - if system: - request_data["system"] = system + if request_system: + request_data["system"] = request_system if temperature is not None: request_data["temperature"] = temperature if thinking: @@ -96,7 +137,6 @@ def _build_responses_kwargs( request_data["output_format"] = output_format anthropic_request: Final = AnthropicMessagesRequest(**request_data) - forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) responses_kwargs: Final = _ADAPTER.translate_request( anthropic_request, include_encrypted_reasoning=_provider_returns_encrypted_reasoning( diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index 642f1857058..e033f1562cd 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -131,12 +131,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter: def _tool_result_output_value( output_text: str, file_parts: tuple[dict[str, str], ...], # mutable-ok: json content parts + tool_result: Mapping[str, object], ) -> str | list[dict[str, str]]: # mutable-ok: API message payload - """Plain string output, or a part list when document file parts are present.""" - if not file_parts: - return output_text + """Plain string output, or a part list when document file parts or a cache breakpoint are present. + + OpenAI reads a breakpoint only on a part, so a marked result ends on a part that carries it. + """ + prompt_cache_breakpoint: Final = tool_result.get("prompt_cache_breakpoint") text_parts: Final = [{"type": "input_text", "text": output_text}] if output_text else [] - return [*text_parts, *file_parts] + parts: Final = [*text_parts, *file_parts] + if not parts or (not file_parts and prompt_cache_breakpoint is None): + return output_text + return [*parts[:-1], with_prompt_cache_breakpoint(parts[-1], prompt_cache_breakpoint)] @staticmethod def _translate_midturn_system_content_to_responses( @@ -327,7 +333,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: { "type": "function_call_output", "call_id": tool_use_id, - "output": self._tool_result_output_value(output_text, tool_file_parts), + "output": self._tool_result_output_value(output_text, tool_file_parts, block), } ) if tool_image_parts: diff --git a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py index 9fad3a2b191..055eec3f6f6 100644 --- a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py @@ -367,6 +367,142 @@ def test_anthropic_sdk_system_string_gets_the_injected_marker(bridge: _Bridge) - bridge.spend.landed(bridge.injecting_off, raw.headers["x-litellm-call-id"], None) +_EPHEMERAL: Final[anthropic.types.CacheControlEphemeralParam] = {"type": "ephemeral"} +_OPTED_IN: Final[Mapping[str, JsonValue]] = {"prompt_cache_options": {"mode": "explicit"}} + + +def _anthropic_message( + client: anthropic.Anthropic, + model: str, + messages: Sequence[anthropic.types.MessageParam], + *, + system: Sequence[anthropic.types.TextBlockParam] = (), + stream: bool = False, +) -> anthropic.types.Message: + if not stream: + return client.messages.create( + model=model, max_tokens=64, system=system, messages=messages, extra_body=dict(pcb.NO_CACHE) + ) + with client.messages.stream( + model=model, max_tokens=64, system=system, messages=messages, extra_body=dict(pcb.NO_CACHE) + ) as events: + return events.get_final_message() + + +def _marked_system() -> list[anthropic.types.TextBlockParam]: + return [ + {"type": "text", "text": "Answer briefly"}, + {"type": "text", "text": "Stable rules", "cache_control": _EPHEMERAL}, + ] + + +def _marked_user(marker: str) -> anthropic.types.MessageParam: + return { + "role": "user", + "content": [ + {"type": "text", "text": pcb.prompt(marker), "cache_control": _EPHEMERAL}, + {"type": "text", "text": "and nothing else"}, + ], + } + + +@pytest.mark.parametrize("stream", [False, True], ids=["unary", "stream"]) +def test_anthropic_sdk_cache_control_reaches_the_responses_wire_as_a_prompt_cache_breakpoint_on_the_same_block( + bridge: _Bridge, stream: bool +) -> None: + marker: Final = uuid.uuid4().hex + with bridge.gateway.scenario() as scenario, _anthropic(bridge) as client: + model: Final = scenario.model(model=pcb.MODEL, api_base=bridge.api_base, **_OPTED_IN) + message: Final = _anthropic_message( + client, model, [_marked_user(marker)], system=_marked_system(), stream=stream + ) + _anthropic_text(message, marker) + body: Final = _wire_body(pcb.posted(bridge.wire, marker), stream=stream) + assert "instructions" not in body, body + assert body["input"] == [ + { + "type": "message", + "role": "developer", + "content": [ + {"type": "input_text", "text": "Answer briefly"}, + {"type": "input_text", "text": "Stable rules", "prompt_cache_breakpoint": pcb.EXPLICIT}, + ], + }, + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": pcb.prompt(marker), "prompt_cache_breakpoint": pcb.EXPLICIT}, + {"type": "input_text", "text": "and nothing else"}, + ], + }, + ], body + + +def test_anthropic_sdk_cache_control_sends_no_mark_to_a_deployment_that_does_not_read_breakpoints( + bridge: _Bridge, +) -> None: + marker: Final = uuid.uuid4().hex + with _anthropic(bridge) as client: + message: Final = _anthropic_message(client, bridge.off, [_marked_user(marker)], system=_marked_system()) + _anthropic_text(message, marker) + request: Final = pcb.posted(bridge.wire, marker) + body: Final = _wire_body(request) + assert body["instructions"] == "Answer briefly\nStable rules", body + assert body["input"] == [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": pcb.prompt(marker)}, + {"type": "input_text", "text": "and nothing else"}, + ], + } + ], body + assert b"cache_control" not in request.body and b"prompt_cache_breakpoint" not in request.body, request.body + + +def test_anthropic_sdk_cache_control_on_a_tool_result_marks_its_function_call_output(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + messages: Final[list[anthropic.types.MessageParam]] = [ + {"role": "user", "content": pcb.prompt(marker)}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "lookup", "input": {}}]}, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "call_1", "content": "found it", "cache_control": _EPHEMERAL} + ], + }, + ] + with bridge.gateway.scenario() as scenario, _anthropic(bridge) as client: + model: Final = scenario.model(model=pcb.MODEL, api_base=bridge.api_base, **_OPTED_IN) + message: Final = _anthropic_message(client, model, messages) + _anthropic_text(message, marker) + request: Final = pcb.posted(bridge.wire, marker) + _wire_body(request) + assert pcb.function_output(pcb.input_items(request), "call_1") == [ + {"type": "input_text", "text": "found it", "prompt_cache_breakpoint": pcb.EXPLICIT} + ], "OpenAI caches up to a breakpoint on a function_call_output part (prompt caching guide, read 2026-10-12)" + + +def test_anthropic_sdk_cache_control_on_an_assistant_turn_is_dropped(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + messages: Final[list[anthropic.types.MessageParam]] = [ + {"role": "user", "content": "first question"}, + {"role": "assistant", "content": [{"type": "text", "text": "earlier answer", "cache_control": _EPHEMERAL}]}, + {"role": "user", "content": pcb.prompt(marker)}, + ] + with bridge.gateway.scenario() as scenario, _anthropic(bridge) as client: + model: Final = scenario.model(model=pcb.MODEL, api_base=bridge.api_base, **_OPTED_IN) + message: Final = _anthropic_message(client, model, messages) + _anthropic_text(message, marker) + request: Final = pcb.posted(bridge.wire, marker) + _wire_body(request) + assert pcb.content_of(pcb.input_items(request), "assistant") == [ + {"type": "output_text", "text": "earlier answer"} + ], "OpenAI accepts a breakpoint on output_text but writes no cache for it (live gpt-5.6, 2026-10-12)" + + def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> None: marker: Final = uuid.uuid4().hex block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)} diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index f596c2adea1..5c2b9412d16 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3070,7 +3070,7 @@ class TestOpenAIPromptCacheBreakpoint: ], ) def test_dialect_resolution(self, model, provider, expected): - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(model, provider) is expected + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, provider) is expected def test_count_covers_both_marker_kinds(self): message = { @@ -3476,8 +3476,8 @@ class TestChatPathProviderStamp: from unittest.mock import patch with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve: - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("gpt-4.1", None) is False + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("my-custom-model", None) is False resolve.assert_not_called() def test_litellm_proxy_target_keeps_anthropic_style_markers(self): @@ -3813,7 +3813,7 @@ class TestHostedOpenAIDialectFlag: def test_flagged_non_openai_deployment_is_eligible(self, monkeypatch): self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") is True ) @@ -3821,7 +3821,7 @@ class TestHostedOpenAIDialectFlag: """The api_base check exists to sniff for api.openai.com, which a Bedrock host never is.""" self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( self.MANTLE_MODEL, "bedrock_mantle", api_base="https://bedrock-runtime.us-east-1.amazonaws.com", @@ -3832,7 +3832,7 @@ class TestHostedOpenAIDialectFlag: def test_flag_set_false_keeps_the_deployment_ineligible(self, monkeypatch): self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=False) assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") is False ) @@ -3840,14 +3840,14 @@ class TestHostedOpenAIDialectFlag: def test_an_odd_typed_flag_keeps_the_deployment_ineligible(self, monkeypatch, flag): self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=flag) assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") is False ) def test_unflagged_non_openai_deployment_stays_ineligible(self, monkeypatch): self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=None) assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") is False ) @@ -3855,7 +3855,7 @@ class TestHostedOpenAIDialectFlag: """A flagged entry does not license a different provider serving the same model string.""" self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "azure") is False + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "azure") is False ) def test_openai_entries_still_go_through_the_api_base_check(self, monkeypatch): @@ -3863,7 +3863,7 @@ class TestHostedOpenAIDialectFlag: assert litellm.model_cost["gpt-5.6"]["supports_prompt_cache_breakpoint"] is True assert litellm.model_cost["gpt-5.6"]["litellm_provider"] == "openai" assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( "gpt-5.6", "openai", api_base="https://some-compatible-host.example.com" ) is False @@ -3871,8 +3871,8 @@ class TestHostedOpenAIDialectFlag: def test_azure_hosted_gpt_5_6_remains_ineligible(self): """Regression guard: the openai entry's flag must not leak to another provider.""" - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-5.6", "azure") is False - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("azure/gpt-5.6", None) is False + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("gpt-5.6", "azure") is False + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("azure/gpt-5.6", None) is False REGIONAL_MODEL = "bedrock_mantle/us-east-1/openai.gpt-5.6-sol" @@ -3880,15 +3880,15 @@ class TestHostedOpenAIDialectFlag: """``bedrock_mantle//`` is a documented routing form the map keys without the region.""" self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle") is True ) - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, None) is True + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, None) is True def test_region_prefixed_deployment_honors_a_flag_set_false(self, monkeypatch): self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=False) assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle") is False ) @@ -3897,31 +3897,31 @@ class TestHostedOpenAIDialectFlag: self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") gov_model = "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-sol" self._register(monkeypatch, gov_model, "bedrock_mantle", flag=None) - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(gov_model, "bedrock_mantle") is False + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(gov_model, "bedrock_mantle") is False def test_region_free_entry_does_not_license_another_provider(self, monkeypatch): """The candidate keys are built for the request's provider, so a flagged Mantle row stays Mantle's.""" self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("us-east-1/openai.gpt-5.6-sol", "azure") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("us-east-1/openai.gpt-5.6-sol", "azure") is False ) def test_unmapped_deployment_name_stays_ineligible(self): - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("azure/my-deployment", None) is False - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-deployment", "azure") is False + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("azure/my-deployment", None) is False + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("my-deployment", "azure") is False def test_bare_name_colliding_with_an_openai_row_reads_its_own_provider_entry(self, monkeypatch): """The Responses layer hands the hook a bare deployment name plus its provider. The openai row keyed by that bare name neither answers for the deployment nor stops the lookup of the provider's own entry.""" self._register(monkeypatch, "gpt-collide", "openai") self._register(monkeypatch, "azure_ai/gpt-collide", "azure_ai") - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is True + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is True def test_bare_name_colliding_with_an_openai_row_stays_ineligible_without_its_own_flag(self, monkeypatch): self._register(monkeypatch, "gpt-collide", "openai") self._register(monkeypatch, "azure_ai/gpt-collide", "azure_ai", flag=None) - assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is False + assert AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is False def test_the_support_check_reads_the_served_providers_row_for_a_provider_stripped_name(self, monkeypatch): """The chat-to-Responses bridge holds the provider-stripped deployment name and the provider that @@ -3962,7 +3962,7 @@ class TestBedrockMantleGptShipsTheOpenAIDialect: def test_shipped_entry_makes_the_deployment_eligible(self): assert supports_openai_prompt_cache_breakpoint(self.MANTLE_MODEL) is True assert ( - AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") is True ) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 1d19264fb9e..c040b161ba4 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -12,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, add_system_prompt_to_messages, + cache_control_as_prompt_cache_breakpoint, encrypted_content_from_signature, encrypted_reasoning_signature, get_file_ids_from_messages, @@ -2263,3 +2264,19 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w messages: list[dict[str, object]], ) -> None: assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) + + +def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None: + marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + + assert cache_control_as_prompt_cache_breakpoint(marked) == { + "type": "text", + "text": "rules", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + + +def test_a_block_without_cache_control_is_returned_untouched() -> None: + unmarked: Final = {"type": "text", "text": "rules"} + + assert cache_control_as_prompt_cache_breakpoint(unmarked) is unmarked diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index eb2d62a8d71..df565ef25b5 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -200,3 +200,76 @@ async def test_streaming_hands_the_logging_object_the_message_id_the_caller_is_s message_start = next(e for e in events if e["type"] == "message_start") assert message_start["message"]["id"].startswith("msg_") assert logging_obj.streamed_anthropic_message_id == message_start["message"]["id"] + + +OPENAI_API: Final = {"custom_llm_provider": "openai", "api_base": "https://api.openai.com/v1"} +OTHER_HOST: Final = {"custom_llm_provider": "openai", "api_base": "https://gateway.example/v1"} +EPHEMERAL: Final = {"type": "ephemeral"} + + +def _request_with_client_cache_marks(cache_control: dict[str, str] | None) -> dict[str, object]: + mark: Final = {} if cache_control is None else {"cache_control": cache_control} + return { + "max_tokens": 1024, + "system": [{"type": "text", "text": "Be concise."}, {"type": "text", "text": "Be helpful.", **mark}], + "messages": [ + {"role": "user", "content": [{"type": "text", "text": "look", **mark}, {"type": "text", "text": "now"}]} + ], + } + + +@pytest.mark.parametrize( + "extra_kwargs", + [ + pytest.param(OPENAI_API, id="api.openai.com"), + pytest.param({**OTHER_HOST, "prompt_cache_options": {"mode": "implicit"}}, id="other-host-opted-in"), + ], +) +def test_client_cache_control_reaches_gpt_5_6_as_a_prompt_cache_breakpoint_on_the_same_block( + extra_kwargs: dict[str, object], +) -> None: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs=extra_kwargs, + **_request_with_client_cache_marks({**EPHEMERAL, "ttl": "1h"}), + ) + + assert "instructions" not in responses_kwargs + assert responses_kwargs["input"] == [ + { + "type": "message", + "role": "developer", + "content": [ + {"type": "input_text", "text": "Be concise."}, + {"type": "input_text", "text": "Be helpful.", "prompt_cache_breakpoint": {"mode": "explicit"}}, + ], + }, + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "look", "prompt_cache_breakpoint": {"mode": "explicit"}}, + {"type": "input_text", "text": "now"}, + ], + }, + ] + + +@pytest.mark.parametrize( + ("model", "extra_kwargs"), + [ + pytest.param("openai/gpt-5.5", OPENAI_API, id="model-older-than-gpt-5.6"), + pytest.param("openai/gpt-5.6", OTHER_HOST, id="other-host-not-opted-in"), + ], +) +def test_client_cache_control_changes_nothing_for_a_deployment_that_does_not_read_prompt_cache_breakpoint( + model: str, extra_kwargs: dict[str, object] +) -> None: + marked: Final = _build_responses_kwargs( + model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(EPHEMERAL) + ) + unmarked: Final = _build_responses_kwargs( + model=model, extra_kwargs=extra_kwargs, **_request_with_client_cache_marks(None) + ) + + assert marked == unmarked diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py index 9b6b44c3d05..1532b38bfae 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -2156,7 +2156,7 @@ class TestPromptCacheBreakpointToResponses: } ] - def test_assistant_and_tool_result_blocks_drop_breakpoint(self): + def test_assistant_blocks_drop_breakpoint_and_a_tool_result_keeps_it_on_its_last_output_part(self): items = _ADAPTER.translate_messages_to_responses_input( [ {"role": "user", "content": [{"type": "text", "text": "q"}]}, @@ -2180,8 +2180,14 @@ class TestPromptCacheBreakpointToResponses: }, ] ) - assert len(items) == 4 - assert not _contains_key(items, "prompt_cache_breakpoint") + *earlier_items, tool_output = items + assert len(earlier_items) == 3 + assert not _contains_key(earlier_items, "prompt_cache_breakpoint") + assert tool_output == { + "type": "function_call_output", + "call_id": "toolu_01", + "output": [{"type": "input_text", "text": "r", "prompt_cache_breakpoint": self.EXPLICIT}], + } def test_prompt_cache_options_forwarded_to_responses_kwargs(self): from litellm.llms.anthropic.pass_through.responses_adapters.handler import (