diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 6af4aef38b0..790588de29a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -26,6 +26,7 @@ from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, + with_prompt_cache_breakpoint, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( @@ -1089,16 +1090,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Handle multimodal content original_type = item.get("type") if original_type == "text": - converted = self._convert_content_str_to_input_text(item.get("text", ""), role) + converted = with_prompt_cache_breakpoint( + self._convert_content_str_to_input_text(item.get("text", ""), role), + item.get("prompt_cache_breakpoint"), + ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) elif original_type == "image_url": # Map to responses API image format - converted = cast( - dict, - self._convert_content_to_responses_format_image( - cast(ChatCompletionImageObject, item), role + converted = with_prompt_cache_breakpoint( + cast( + dict, + self._convert_content_to_responses_format_image( + cast(ChatCompletionImageObject, item), role + ), ), + item.get("prompt_cache_breakpoint"), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) @@ -1110,8 +1117,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): result.append(converted) verbose_logger.debug("Chat provider: image -> %s", converted) elif item_type == "file": - converted = _input_file_from_file_value( - cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked + converted = with_prompt_cache_breakpoint( + _input_file_from_file_value( + cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked + ), + item.get("prompt_cache_breakpoint"), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4487aa31e3b..893cb0d84d3 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -283,7 +283,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): openai_dialect and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before ): - non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit")) + non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="implicit")) # Points this pass did not place: non-message ones for the provider transform, and # the deferred role-targeted ones. Deferring is what reaches the Responses API's diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 4d89f021fe9..d2ddb4dc9ac 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -4357,6 +4357,53 @@ def _system_input_item(text: str) -> dict[str, object]: return {"type": "message", "role": "system", "content": [{"type": "input_text", "text": text}]} +@pytest.mark.parametrize( + ("content_block", "expected_content"), + [ + ( + {"type": "text", "text": "Stable prefix"}, + {"type": "input_text", "text": "Stable prefix"}, + ), + ( + {"type": "image_url", "image_url": "https://example.com/image.png"}, + {"type": "input_image", "image_url": "https://example.com/image.png", "detail": "auto"}, + ), + ( + {"type": "file", "file": {"file_id": "file-123"}}, + {"type": "input_file", "file_id": "file-123"}, + ), + ], + ids=("text", "image_url", "file"), +) +def test_prompt_cache_breakpoint_survives_chat_to_responses_conversion( + content_block: dict[str, object], expected_content: dict[str, object] +) -> None: + handler: Final = LiteLLMResponsesTransformationHandler() + cache_breakpoint: Final = {"mode": "explicit"} + marked_content: Final = {**content_block, "prompt_cache_breakpoint": cache_breakpoint} + + request: Final = handler.transform_request( + model="gpt-5.6-sol", + messages=[ + { + "role": "user", + "content": [marked_content], + } + ], + optional_params={"prompt_cache_options": cache_breakpoint}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert request["input"][0] == { + "type": "message", + "role": "user", + "content": [{**expected_content, "prompt_cache_breakpoint": cache_breakpoint}], + } + assert request["prompt_cache_options"] == cache_breakpoint + + def test_mid_conversation_system_string_stays_in_input_after_a_user_turn(): handler: Final = LiteLLMResponsesTransformationHandler() diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 70039646b4b..23fa2e36bce 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -3208,7 +3208,7 @@ class TestOpenAIPromptCacheBreakpoint: } assert processed[1] == {"role": "user", "content": "hi"} assert returned is params - assert returned == {"prompt_cache_options": self.EXPLICIT} + assert returned == {"prompt_cache_options": {"mode": "implicit"}} def test_chat_list_content_marks_last_block(self): messages = [ @@ -3230,21 +3230,22 @@ class TestOpenAIPromptCacheBreakpoint: "prompt_cache_breakpoint": self.EXPLICIT, }, ] - assert params["prompt_cache_options"] == self.EXPLICIT + assert params["prompt_cache_options"] == {"mode": "implicit"} def test_chat_unprefixed_model_resolves_to_openai(self): params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} _, processed, _ = self._chat([{"role": "system", "content": "sys"}], params, model="gpt-5.6") assert processed[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] - assert params["prompt_cache_options"] == self.EXPLICIT + assert params["prompt_cache_options"] == {"mode": "implicit"} - def test_chat_keeps_caller_prompt_cache_options(self): + @pytest.mark.parametrize("mode", ["explicit", "implicit"]) + def test_chat_keeps_caller_prompt_cache_options(self, mode): params = { "cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT), - "prompt_cache_options": {"mode": "implicit"}, + "prompt_cache_options": {"mode": mode, "ttl": "24h"}, } self._chat([{"role": "system", "content": "sys"}], params) - assert params["prompt_cache_options"] == {"mode": "implicit"} + assert params["prompt_cache_options"] == {"mode": mode, "ttl": "24h"} def test_chat_no_prompt_cache_options_when_nothing_injected(self): params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)} @@ -3278,7 +3279,7 @@ class TestOpenAIPromptCacheBreakpoint: _, processed, _ = self._chat(messages, params) assert processed[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}] assert processed[1] == messages[1] - assert params["prompt_cache_options"] == self.EXPLICIT + assert params["prompt_cache_options"] == {"mode": "implicit"} def test_cap_counts_client_breakpoints_of_both_kinds(self): messages = [ @@ -3330,7 +3331,7 @@ class TestOpenAIPromptCacheBreakpointPlacementRules: ] out, params = self._chat(messages, [{"location": "message", "index": -1}]) assert out[2]["content"] == [{"type": "text", "text": "sunny", "prompt_cache_breakpoint": self.EXPLICIT}] - assert params["prompt_cache_options"] == self.EXPLICIT + assert params["prompt_cache_options"] == {"mode": "implicit"} def test_tool_result_only_turn_is_skipped_on_v1_messages(self): messages = [ @@ -3442,12 +3443,12 @@ class TestChatPathProviderStamp: def test_explicit_openai_provider_uses_openai_dialect(self): out, params = self._seed_and_run("gpt-5.6", "openai") assert out[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] - assert params["prompt_cache_options"] == {"mode": "explicit"} + assert params["prompt_cache_options"] == {"mode": "implicit"} def test_bare_gpt_model_without_provider_resolves_to_openai(self): out, params = self._seed_and_run("gpt-5.6", None) assert out[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] - assert params["prompt_cache_options"] == {"mode": "explicit"} + assert params["prompt_cache_options"] == {"mode": "implicit"} def test_points_keep_identity_for_models_below_gpt_5_6(self): points = copy.deepcopy(self.POINTS) @@ -3488,7 +3489,7 @@ class TestChatPathProviderStamp: def test_regional_openai_api_base_uses_openai_dialect(self): out, params = self._seed_and_run("gpt-5.6", None, api_base="https://eu.api.openai.com/v1") assert out[0]["content"] == self.OPENAI_STYLE - assert params["prompt_cache_options"] == {"mode": "explicit"} + assert params["prompt_cache_options"] == {"mode": "implicit"} @pytest.mark.parametrize("env_var", ["OPENAI_BASE_URL", "OPENAI_API_BASE"]) def test_env_api_base_override_keeps_anthropic_style_markers(self, monkeypatch, env_var): @@ -3507,7 +3508,7 @@ class TestChatPathProviderStamp: monkeypatch.setenv("OPENAI_BASE_URL", self.CUSTOM_API_BASE) out, params = self._seed_and_run("gpt-5.6", None, api_base="https://api.openai.com/v1") assert out[0]["content"] == self.OPENAI_STYLE - assert params["prompt_cache_options"] == {"mode": "explicit"} + assert params["prompt_cache_options"] == {"mode": "implicit"} @pytest.mark.parametrize( "api_base,expected", @@ -3598,7 +3599,7 @@ class TestResponsesInputPartsEligible: "text": "second", "prompt_cache_breakpoint": self.EXPLICIT, } - assert params["prompt_cache_options"] == self.EXPLICIT + assert params["prompt_cache_options"] == {"mode": "implicit"} @pytest.mark.parametrize( "part", @@ -3610,7 +3611,7 @@ class TestResponsesInputPartsEligible: def test_input_image_and_input_file_parts_are_eligible(self, part): out, params = self._chat([{"role": "user", "content": [part]}], [{"location": "message", "index": -1}]) assert out[0]["content"][0] == {**part, "prompt_cache_breakpoint": self.EXPLICIT} - assert params["prompt_cache_options"] == self.EXPLICIT + assert params["prompt_cache_options"] == {"mode": "implicit"} class TestMessagesPathApiBaseGate: @@ -3681,7 +3682,7 @@ class TestToolConfigSlotInOpenAIDialect: dynamic_callback_params={}, ) assert [msg["content"][0].get("prompt_cache_breakpoint") for msg in out] == [self.EXPLICIT] * 4 - assert params["prompt_cache_options"] == self.EXPLICIT + assert params["prompt_cache_options"] == {"mode": "implicit"} def test_messages_path_marks_all_four_messages(self): out, _, _ = AnthropicCacheControlHook.apply_to_anthropic_messages_request( @@ -3744,7 +3745,7 @@ class TestPromptCacheBreakpointCapability: assert chat_messages[0]["content"] == [ {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} ] - assert chat_params["prompt_cache_options"] == {"mode": "explicit"} + assert chat_params["prompt_cache_options"] == {"mode": "implicit"} kwargs = {"cache_control_injection_points": copy.deepcopy(points)} _, system = AnthropicCacheControlHook.maybe_inject_cache_control( diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index b27401d693a..3ef3cd8854a 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -662,7 +662,7 @@ async def test_aresponses_injection_point_marks_input_text_on_gpt_5_6(): "prompt_cache_breakpoint": {"mode": "explicit"}, } assert body["input"][1] == {"role": "user", "content": "hi"} - assert body["prompt_cache_options"] == {"mode": "explicit"} + assert body["prompt_cache_options"] == {"mode": "implicit"} def test_responses_injection_point_marks_input_text_on_gpt_5_6(): @@ -683,7 +683,7 @@ def test_responses_injection_point_marks_input_text_on_gpt_5_6(): "prompt_cache_breakpoint": {"mode": "explicit"}, } assert body["input"][1] == {"role": "user", "content": "hi"} - assert body["prompt_cache_options"] == {"mode": "explicit"} + assert body["prompt_cache_options"] == {"mode": "implicit"} @pytest.mark.asyncio @@ -767,7 +767,7 @@ async def test_aresponses_custom_api_base_opts_in_through_prompt_cache_options() async def test_aresponses_regional_openai_api_base_marks_input_text(): body = await _aresponses_body_with_system_point(model="gpt-5.6", api_base="https://eu.api.openai.com/v1") assert body["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"} - assert body["prompt_cache_options"] == {"mode": "explicit"} + assert body["prompt_cache_options"] == {"mode": "implicit"} @pytest.mark.usefixtures("_no_openai_api_base_override") diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 3f484592418..cff1f5a8535 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -3658,7 +3658,7 @@ def test_completion_default_api_base_sends_prompt_cache_breakpoint_for_gpt_5_6() assert request_body["messages"][0]["content"] == [ {"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}} ] - assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "explicit"} + assert request_body["extra_body"]["prompt_cache_options"] == {"mode": "implicit"} _SUBSCRIPTION_OAUTH_CREDENTIAL = "Bearer sk-ant-oat01-fake-subscription-token-for-testing-0123456789"