From f2c6adefc52908eb8162ffaa0b1893ed51618560 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 14:13:19 -0700 Subject: [PATCH] fix(responses): drop a malformed prompt_cache_breakpoint under drop_params in the chat-to-Responses bridge --- .../transformation.py | 41 +++++++++--- ...responses_transformation_transformation.py | 64 +++++++++++++++++++ 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 82bd83621f0..c631d9f629e 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -19,7 +19,7 @@ from openai.types.responses.response_input_param import ( from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm import ModelResponse @@ -45,6 +45,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, + PromptCacheBreakpoint, Reasoning, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, @@ -205,6 +206,19 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li return "length" +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + try: + return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + + def _input_file_from_file_value(file_value: object) -> dict[str, object]: if not isinstance(file_value, dict): return {"type": "input_file"} @@ -364,7 +378,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return None, index def convert_chat_completion_messages_to_responses_api( - self, messages: list["AllMessageValues"] + self, messages: list["AllMessageValues"], drop_params: bool = False ) -> tuple[list[object], str | None]: input_items: Final[list[object]] = [] instructions: str | None = None @@ -405,6 +419,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): "content": self._convert_content_to_responses_format( content, role, + drop_params=drop_params, ), } ) @@ -423,6 +438,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tool_output = self._convert_content_to_responses_format( content, "user", # Use "user" role to get input_* types + drop_params=drop_params, ) else: # Fallback: convert unexpected types to input_text @@ -450,7 +466,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): { "type": "message", "role": "assistant", - "content": self._convert_content_to_responses_format(content, "assistant"), + "content": self._convert_content_to_responses_format( + content, "assistant", drop_params=drop_params + ), } ) for tool_call in tool_calls: @@ -484,7 +502,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): { "type": "message", "role": role, - "content": self._convert_content_to_responses_format(content, cast(str, role)), + "content": self._convert_content_to_responses_format( + content, cast(str, role), drop_params=drop_params + ), } ) elif role == "assistant": @@ -597,7 +617,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ( input_items, instructions, - ) = self.convert_chat_completion_messages_to_responses_api(messages) + ) = self.convert_chat_completion_messages_to_responses_api( + messages, drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params) + ) # OpenAI's Responses API rejects an empty input. For a system-only # request, carry the system message as a system-role input item instead @@ -1072,6 +1094,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ] | None, role: str, + drop_params: bool = False, ) -> list[dict[str, object]]: """Convert chat completion content to responses API format""" from litellm.types.llms.openai import ChatCompletionImageObject @@ -1098,7 +1121,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if original_type == "text": converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(item.get("text", ""), role), - item.get("prompt_cache_breakpoint"), + _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) @@ -1111,7 +1134,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(ChatCompletionImageObject, item), role ), ), - item.get("prompt_cache_breakpoint"), + _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) @@ -1127,7 +1150,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): _input_file_from_file_value( cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked ), - item.get("prompt_cache_breakpoint"), + _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) @@ -1151,7 +1174,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Default to input_text for unknown types converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(str(item.get("text", item)), role), - item.get("prompt_cache_breakpoint"), + _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index dff408b738c..30596599066 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -4642,3 +4642,67 @@ def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_br content = response[0]["content"] assert [block["type"] for block in content] == ["input_text", "input_text"] assert content[1]["prompt_cache_breakpoint"] == breakpoint_marker + + +_HAND_WRITTEN_PROMPT_CACHE_BREAKPOINT_BLOCKS: Final = ( + {"type": "text", "text": "a string marker", "prompt_cache_breakpoint": "explicit"}, + {"type": "text", "text": "an unknown mode", "prompt_cache_breakpoint": {"mode": "bogus"}}, + { + "type": "input_audio", + "input_audio": {"data": "Zm9v", "format": "wav"}, + "prompt_cache_breakpoint": ["explicit"], + }, + {"type": "text", "text": "an extra key", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}, + {"type": "text", "text": "well formed", "prompt_cache_breakpoint": {"mode": "explicit"}}, +) + + +def test_convert_chat_completion_messages_to_responses_api_drops_malformed_prompt_cache_breakpoint_under_drop_params(): + """Under drop_params a marker the Responses API would reject is dropped like any other unsupported value. + + A well-formed marker still reaches the wire, and an extra key is dropped from an otherwise valid one. + """ + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": list(_HAND_WRITTEN_PROMPT_CACHE_BREAKPOINT_BLOCKS)}] + + response, _ = handler.convert_chat_completion_messages_to_responses_api(messages, drop_params=True) + + content = response[0]["content"] + assert [block.get("prompt_cache_breakpoint") for block in content] == [ + None, + None, + None, + {"mode": "explicit"}, + {"mode": "explicit"}, + ] + assert all("prompt_cache_breakpoint" not in block for block in content[:3]) + + +def test_convert_chat_completion_messages_to_responses_api_keeps_malformed_prompt_cache_breakpoint_by_default(): + """Without drop_params every marker goes out as written, and the provider judges a malformed one.""" + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": list(_HAND_WRITTEN_PROMPT_CACHE_BREAKPOINT_BLOCKS)}] + + response, _ = handler.convert_chat_completion_messages_to_responses_api(messages) + + content = response[0]["content"] + assert [block["prompt_cache_breakpoint"] for block in content] == [ + block["prompt_cache_breakpoint"] for block in _HAND_WRITTEN_PROMPT_CACHE_BREAKPOINT_BLOCKS + ] + + +def test_transform_request_drop_params_in_litellm_params_gates_the_prompt_cache_breakpoint_carry(): + """The per-request drop_params flag travels in litellm_params and has to reach the content converter.""" + handler = LiteLLMResponsesTransformationHandler() + messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": "explicit"}]}] + + result = handler.transform_request( + model="gpt-6.1-sol", + messages=messages, + optional_params={}, + litellm_params={"drop_params": True}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0]