diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 3c1b3cf964b..f92230f5755 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,6 +13,8 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast +from pydantic import TypeAdapter + import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -1805,6 +1807,33 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: ) +_BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) +_BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) +_BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) + + +def _without_prompt_cache_breakpoint(value: object) -> object: + if not isinstance(value, Mapping): + return value + fields: Final = _BREAKPOINT_FIELDS.validate_python(value) + return {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"} + + +def _breakpoints_stripped_from(key: str, value: object) -> object: + if key in _BREAKPOINT_BLOCK_LIST_KEYS and isinstance(value, list): + return [_without_prompt_cache_breakpoint(block) for block in _BREAKPOINT_BLOCKS.validate_python(value)] + return _without_prompt_cache_breakpoint(value) if key == "function" else value + + +def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT: + """A message or tool definition with every ``prompt_cache_breakpoint`` removed: its own, its content parts', + its tool calls' and its ``function``'s.""" + stripped: Final = { + key: _breakpoints_stripped_from(key, value) for key, value in block.items() if key != "prompt_cache_breakpoint" + } + return cast(_MarkedT, stripped) # cast-ok: same block shape as the input minus the marker keys + + LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"}) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index a51704085bd..584cecc0f36 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -28,6 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( hoist_images_from_tool_messages, system_messages_first, tool_with_sanitized_parameters, + without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, @@ -474,7 +475,8 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on api.openai.com, or one opted in through ``prompt_cache_options`` or the cost map) gets each Anthropic ``cache_control`` mark rewritten in place. Any other - endpoint keeps or strips the marks exactly as before. + endpoint keeps or strips the marks as before, and under drop_params also loses + every ``prompt_cache_breakpoint`` it cannot read. """ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook @@ -485,14 +487,24 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base): return messages, tools + drops_breakpoints: Final = not reads_breakpoints and ( + litellm.drop_params is True or litellm_params.get("drop_params") is True + ) renamed: Final = ( [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages] if reads_breakpoints + else [without_prompt_cache_breakpoints(message) for message in messages] + if drops_breakpoints else list(messages) ) - return self.remove_cache_control_flag_from_messages_and_tools( - model=model, messages=renamed, tools=list(tools) if tools is not None else None + kept_tools: Final = ( + None + if tools is None + else [without_prompt_cache_breakpoints(tool) for tool in tools] + if drops_breakpoints + else list(tools) ) + return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools) def _sanitized_tools_update_for_openai( self, diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index a0a55df95e6..76faf338307 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1592,6 +1592,43 @@ class TestOpenAIPromptCacheBreakpointChatPath: assert request["extra_body"] == {"prompt_cache_options": self.EXPLICIT} assert "prompt_cache_options" not in request + @pytest.mark.parametrize( + ("model", "drop_params", "sent"), + [("gpt-4.1", True, False), ("gpt-4.1", False, True), ("gpt-5.6", True, True)], + ids=["unread-dropped", "unread-sent-through", "read-kept"], + ) + def test_drop_params_strips_breakpoints_only_where_the_model_cannot_read_them( + self, model: str, drop_params: bool, sent: bool + ): + marked_part: Final = {"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT} + part: Final = {"type": "text", "text": "hi"} + call: Final = {"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}} + function: Final = {"name": "f", "parameters": {"type": "object"}} + marked: Final = {"prompt_cache_breakpoint": self.EXPLICIT} + + request = OpenAIGPTConfig().transform_request( + model=model, + messages=[ + {"role": "user", "content": [marked_part]}, + {"role": "assistant", "content": "ok", **marked, "tool_calls": [{**call, **marked}]}, + {"role": "tool", "tool_call_id": "c1", "content": "done"}, + ], + optional_params={"tools": [{"type": "function", "function": {**function, **marked}}, {"type": "function", "function": function, **marked}]}, + litellm_params={"custom_llm_provider": "openai", "drop_params": drop_params}, + headers={}, + ) + + mark: Final = marked if sent else {} + assert request["messages"] == [ + {"role": "user", "content": [marked_part if sent else part]}, + {"role": "assistant", "content": "ok", **mark, "tool_calls": [{**call, **mark}]}, + {"role": "tool", "tool_call_id": "c1", "content": "done"}, + ] + assert request["tools"] == [ + {"type": "function", "function": {**function, **mark}}, + {"type": "function", "function": function, **mark}, + ] + _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$'