From c270a243332e1963a2ab3ad80300a07d29f6cdb9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:23:25 -0700 Subject: [PATCH 1/7] fix(responses): drop a malformed prompt_cache_breakpoint under drop_params on the direct /v1/responses path --- .../prompt_templates/common_utils.py | 16 +++ .../llms/openai/responses/transformation.py | 61 +++++++++- ...ses_bridge_prompt_cache_breakpoint_wire.py | 44 ++++--- .../test_openai_responses_transformation.py | 107 ++++++++++++++++++ 4 files changed, 211 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 0243141eda2..a9c124755f8 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,6 +13,8 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast +from pydantic import TypeAdapter, ValidationError + import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -29,6 +31,7 @@ from litellm.types.llms.openai import ( ChatCompletionThinkingBlock, ChatCompletionToolParam, ChatCompletionUserMessage, + PromptCacheBreakpoint, ) from litellm.types.utils import ( Choices, @@ -1793,6 +1796,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + try: + return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + + LITELLM_INTERNAL_MESSAGE_FIELDS: Final = frozenset({"thinking_blocks", "reasoning_content", "provider_specific_fields"}) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index ec4f0c04475..b80c82d26f4 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -18,6 +18,8 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_non_python_regex_patterns, flatten_combinators_and_drop_non_python_regex_patterns, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -37,7 +39,8 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 -_RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) +_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) +_JSON_ARRAY: Final = TypeAdapter(list[object], config=ConfigDict(strict=True)) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -51,6 +54,21 @@ _MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS: Final = ("gpt-4", "gpt-3 _PROVIDERS_WITH_OPENAI_SCHEMA_VALIDATOR: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI}) _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, LlmProviders.OPENAI}) _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS +_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"}) + + +def _json_object(value: object) -> Mapping[str, object] | None: + try: + return _JSON_OBJECT.validate_python(value) + except ValidationError: + return None + + +def _json_array(value: object) -> Sequence[object] | None: + try: + return _JSON_ARRAY.validate_python(value) + except ValidationError: + return None class _ReasoningSupportEntry(LiteLLMBaseModel): @@ -326,11 +344,46 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( model=model, input=validated_input, tools=tools ) + marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints( + input=stripped_input, litellm_params=litellm_params + ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params ) - return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools + return self._drop_foreign_tool_call_item_ids(marker_safe_input), sanitized_tools + + def _input_with_wire_safe_prompt_cache_breakpoints( + self, input: str | ResponseInputParam, litellm_params: GenericLiteLLMParams + ) -> str | ResponseInputParam: + deployment_drop_params: Final[object] = getattr(litellm_params, "drop_params", None) + if not (deployment_drop_params or litellm.drop_params) or not isinstance(input, list): + return input + wire_safe_items: Final = [self._item_with_wire_safe_prompt_cache_breakpoints(item) for item in input] + return cast("ResponseInputParam", wire_safe_items) # cast-ok: items keep their shape, minus a rejected marker + + @classmethod + def _item_with_wire_safe_prompt_cache_breakpoints(cls, item: object) -> object: + wire_item: Final = _json_object(item) + if wire_item is None: + return item + return {key: cls._parts_with_wire_safe_prompt_cache_breakpoints(key, value) for key, value in wire_item.items()} + + @classmethod + def _parts_with_wire_safe_prompt_cache_breakpoints(cls, key: str, value: object) -> object: + parts: Final = _json_array(value) if key in _INPUT_PART_LIST_KEYS else None + if parts is None: + return value + return [cls._part_with_wire_safe_prompt_cache_breakpoint(part) for part in parts] + + @staticmethod + def _part_with_wire_safe_prompt_cache_breakpoint(part: object) -> object: + wire_part: Final = _json_object(part) + if wire_part is None or "prompt_cache_breakpoint" not in wire_part: + return part + marker: Final = prompt_cache_breakpoint_for_wire(wire_part["prompt_cache_breakpoint"], drop_params=True) + unmarked: Final = {key: value for key, value in wire_part.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, marker) def _tools_with_object_parameters( self, model: str, tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None @@ -578,7 +631,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): original_response=raw_response.text, additional_args={"complete_input_dict": {}}, ) - raw_response_json: Final = _RAW_RESPONSE_JSON.validate_python(raw_response.json()) + raw_response_json: Final = _JSON_OBJECT.validate_python(raw_response.json()) raw_response_json["created_at"] = safe_convert_created_field(raw_response_json["created_at"]) except Exception: raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code) @@ -978,7 +1031,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): original_response=raw_response.text, additional_args={"complete_input_dict": {}}, ) - raw_response_json: Final = _RAW_RESPONSE_JSON.validate_python(raw_response.json()) + raw_response_json: Final = _JSON_OBJECT.validate_python(raw_response.json()) raw_response_json["created_at"] = safe_convert_created_field(raw_response_json["created_at"]) except Exception: raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code) diff --git a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py index 9fad3a2b191..ede82c14e10 100644 --- a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py @@ -39,14 +39,6 @@ _CASES: Final[tuple[tuple[str, Mode, JsonValue, JsonValue], ...]] = ( ) _CASE_IDS: Final = tuple(case[0] for case in _CASES) _CASE_VALUES: Final = tuple(case[1:] for case in _CASES) -_ADAPTER_CASES: Final[tuple[tuple[str, Mode, JsonValue, JsonValue], ...]] = ( - ("valid-on", "on", pcb.EXPLICIT, pcb.EXPLICIT), - ("valid-off", "off", pcb.EXPLICIT, pcb.EXPLICIT), - ("malformed-on", "on", "yes", "yes"), - ("malformed-off", "off", "yes", "yes"), -) -_ADAPTER_CASE_IDS: Final = tuple(case[0] for case in _ADAPTER_CASES) -_ADAPTER_CASE_VALUES: Final = tuple(case[1:] for case in _ADAPTER_CASES) @dataclass(frozen=True, slots=True) @@ -331,7 +323,7 @@ def _anthropic_text(message: anthropic.types.Message, marker: str) -> None: assert content.type == "text" and content.text == rv.answer(marker), message -@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _ADAPTER_CASE_VALUES, ids=_ADAPTER_CASE_IDS) +@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS) def test_anthropic_sdk_marker_on_user_text( bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue ) -> None: @@ -367,14 +359,17 @@ def test_anthropic_sdk_system_string_gets_the_injected_marker(bridge: _Bridge) - bridge.spend.landed(bridge.injecting_off, raw.headers["x-litellm-call-id"], None) -def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> None: +@pytest.mark.parametrize( + ("mode", "expected"), (("on", pcb.EXPLICIT), ("off", _UNKNOWN_KEY)), ids=("normalized-on", "verbatim-off") +) +def test_native_responses_marker_with_an_unknown_key(bridge: _Bridge, mode: Mode, expected: JsonValue) -> None: marker: Final = uuid.uuid4().hex block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)} response: Final = bridge.gateway.request( "POST", "/v1/responses", { - "model": bridge.on, + "model": bridge.model(mode), "input": [{"type": "message", "role": "user", "content": [pcb.marked(block, _UNKNOWN_KEY)]}], **pcb.NO_CACHE, }, @@ -386,8 +381,31 @@ def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> No request: Final = pcb.posted(bridge.wire, marker) _wire_body(request) on_wire: Final = pcb.single_block(pcb.input_items(request), "user") - assert on_wire == pcb.marked(block, _UNKNOWN_KEY), on_wire - bridge.spend.landed(bridge.on, response.headers["x-litellm-call-id"], marker) + assert on_wire == pcb.marked(block, expected), on_wire + bridge.spend.landed(bridge.model(mode), response.headers["x-litellm-call-id"], marker) + + +@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS) +def test_openai_sdk_native_responses_marker_on_user_text( + bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue +) -> None: + marker: Final = uuid.uuid4().hex + block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)} + with openai.OpenAI(api_key=bridge.gateway.key, base_url=_v1(bridge.gateway), max_retries=0) as client: + raw: Final = client.responses.with_raw_response.create( + model=bridge.model(mode), + input=[{"type": "message", "role": "user", "content": [pcb.marked(block, breakpoint)]}], + extra_body=dict(pcb.NO_CACHE), + ) + response: Final = raw.parse() + assert pcb.answers(response.id, marker), response + assert response.output_text == rv.answer(marker), response + request: Final = pcb.posted(bridge.wire, marker) + _wire_body(request) + on_wire: Final = pcb.single_block(pcb.input_items(request), "user") + assert on_wire["type"] == "input_text" and on_wire["text"] == pcb.prompt(marker), on_wire + pcb.assert_marker(on_wire, expected) + bridge.spend.landed(bridge.model(mode), raw.headers["x-litellm-call-id"], marker) @pytest.mark.parametrize("breakpoint", _MALFORMED_VALUES, ids=_MALFORMED_IDS) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index a8f0d23e560..509cfec5075 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -19,6 +19,7 @@ from litellm.types.llms.openai import ( OutputTextDeltaEvent, ResponseAPIUsage, ResponseCompletedEvent, + ResponseInputParam, ResponsesAPIResponse, ResponsesAPIStreamEvents, ) @@ -3445,3 +3446,109 @@ async def test_extra_body_merges_with_request_data(extra_body_mock_response_data assert "temperature" in request_body assert "custom_field" in request_body assert request_body["custom_field"] == "custom_value" + + +_MALFORMED_MARKERS: Final = ("yes", 1, {"mode": "bogus"}, {"mode": "explicit", "ttl": "1h"}) +_MALFORMED_MARKER_IDS: Final = ("string", "int", "bogus-mode", "bad-ttl") +_VALID_MARKERS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"}) +_VALID_MARKER_IDS: Final = ("explicit", "explicit-30m") +_PARTS: Final[Mapping[str, Mapping[str, object]]] = { + "input_text": {"type": "input_text", "text": "Say ok"}, + "input_image": {"type": "input_image", "image_url": "https://example.com/a.png"}, + "input_file": {"type": "input_file", "file_id": "file-abc"}, +} +_CONFIGS: Final = (OpenAIResponsesAPIConfig(), AzureOpenAIResponsesAPIConfig()) +_CONFIG_IDS: Final = ("openai", "azure") + + +def _marked_message(part_type: str, marker: object) -> list[dict[str, object]]: + return [{"type": "message", "role": "user", "content": [{**_PARTS[part_type], "prompt_cache_breakpoint": marker}]}] + + +def _transformed_input( + config: BaseResponsesAPIConfig, input: str | list[dict[str, object]], litellm_params: GenericLiteLLMParams +) -> object: + request: Final = config.transform_responses_api_request( + model="gpt-6.1-sol", + input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items + response_api_optional_request_params={}, + litellm_params=litellm_params, + headers={}, + ) + return request["input"] + + +def _single_part(transformed_input: object) -> Mapping[str, object]: + (item,) = cast(list[Mapping[str, object]], transformed_input) + (part,) = cast(list[Mapping[str, object]], item["content"]) + return part + + +@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS) +@pytest.mark.parametrize("part_type", tuple(_PARTS)) +@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS) +def test_malformed_prompt_cache_breakpoint_is_dropped_under_deployment_drop_params( + monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig, part_type: str, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + transformed: Final = _transformed_input( + config, _marked_message(part_type, marker), GenericLiteLLMParams(drop_params=True) + ) + assert _single_part(transformed) == _PARTS[part_type] + + +@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS) +def test_malformed_prompt_cache_breakpoint_is_dropped_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + transformed: Final = _transformed_input(_CONFIGS[0], _marked_message("input_text", marker), GenericLiteLLMParams()) + assert _single_part(transformed) == _PARTS["input_text"] + + +@pytest.mark.parametrize("drop_params", (False, None), ids=("false", "null")) +@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS) +def test_malformed_prompt_cache_breakpoint_is_forwarded_verbatim_without_drop_params( + monkeypatch: pytest.MonkeyPatch, marker: object, drop_params: bool | None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", marker) + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=drop_params)) == input + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +@pytest.mark.parametrize("marker", _VALID_MARKERS, ids=_VALID_MARKER_IDS) +def test_valid_prompt_cache_breakpoint_reaches_the_wire_under_both_settings( + monkeypatch: pytest.MonkeyPatch, marker: Mapping[str, str], drop_params: bool +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", marker) + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=drop_params)) == input + + +def test_prompt_cache_breakpoint_unknown_key_is_trimmed_under_drop_params(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", {"mode": "explicit", "note": "kept"}) + dropped: Final = _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) + assert _single_part(dropped)["prompt_cache_breakpoint"] == {"mode": "explicit"} + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=False)) == input + + +def test_malformed_prompt_cache_breakpoint_on_a_tool_output_part_is_dropped_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + marked_output: Final = [{"type": "input_text", "text": "42", "prompt_cache_breakpoint": "yes"}] + input: Final[list[dict[str, object]]] = [{"type": "function_call_output", "call_id": "call_1", "output": marked_output}] + (item,) = cast( + list[Mapping[str, object]], _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) + ) + assert item["output"] == [{"type": "input_text", "text": "42"}] + + +@pytest.mark.parametrize("input", ("hi", [{"type": "message", "role": "user", "content": "hi"}]), ids=("string", "unmarked")) +def test_input_without_a_marker_is_untouched_under_drop_params( + monkeypatch: pytest.MonkeyPatch, input: str | list[dict[str, object]] +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) == input From 458705c0e7c5350b3561b6d134430a8a65574d4b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:42:54 -0700 Subject: [PATCH 2/7] refactor(responses): reuse the shared prompt_cache_breakpoint_for_wire helper in the chat bridge --- .../transformation.py | 27 +++-------- ...ore_utils_prompt_templates_common_utils.py | 47 +++++++++++++++++++ 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 41f6a211036..1813389c685 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -20,7 +20,7 @@ from openai.types.responses.response_input_param import ( from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam -from pydantic import BaseModel, TypeAdapter, ValidationError +from pydantic import BaseModel import litellm from litellm import ModelResponse @@ -28,6 +28,7 @@ from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( + prompt_cache_breakpoint_for_wire, responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) @@ -47,7 +48,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, - PromptCacheBreakpoint, Reasoning, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, @@ -317,19 +317,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li return "length" -_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) - - -def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - try: - return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) - except ValidationError: - verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker) - return None - - def _input_file_from_file_value(file_value: object) -> dict[str, object]: if not isinstance(file_value, dict): return {"type": "input_file"} @@ -1260,7 +1247,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if original_type == "text": converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(item.get("text", ""), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) @@ -1273,14 +1260,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(ChatCompletionImageObject, item), role ), ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) elif original_type == "input_audio": converted = with_prompt_cache_breakpoint( {"type": "input_audio", "input_audio": item.get("input_audio")}, - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: input_audio -> %s", converted) @@ -1296,7 +1283,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): _input_file_from_file_value( cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) @@ -1320,7 +1307,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Default to input_text for unknown types converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(str(item.get("text", item)), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 1d19264fb9e..ed47dee8f2b 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_encrypted_reasoning_block, merge_consecutive_system_messages, parse_tool_call_arguments, + prompt_cache_breakpoint_for_wire, responses_reasoning_items_from_thinking_blocks, salvage_concatenated_tool_arguments, split_concatenated_json_objects, @@ -2263,3 +2264,49 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w messages: list[dict[str, object]], ) -> None: assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) + + +_MALFORMED_PROMPT_CACHE_BREAKPOINTS: Final = ( + "yes", + 1, + ["explicit"], + "", + {}, + {"mode": "bogus"}, + {"mode": "explicit", "ttl": "1h"}, +) +_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS: Final = ( + "string", + "int", + "list", + "empty-string", + "empty-object", + "bogus-mode", + "bad-ttl", +) +_VALID_PROMPT_CACHE_BREAKPOINTS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"}) + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +def test_prompt_cache_breakpoint_for_wire_passes_a_missing_marker_through(drop_params: bool) -> None: + assert prompt_cache_breakpoint_for_wire(None, drop_params) is None + + +@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) +def test_prompt_cache_breakpoint_for_wire_drops_a_malformed_marker_under_drop_params(marker: object) -> None: + assert prompt_cache_breakpoint_for_wire(marker, True) is None + + +@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) +def test_prompt_cache_breakpoint_for_wire_forwards_a_malformed_marker_without_drop_params(marker: object) -> None: + assert prompt_cache_breakpoint_for_wire(marker, False) is marker + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +@pytest.mark.parametrize("marker", _VALID_PROMPT_CACHE_BREAKPOINTS, ids=("explicit", "explicit-30m")) +def test_prompt_cache_breakpoint_for_wire_keeps_a_valid_marker(marker: dict[str, str], drop_params: bool) -> None: + assert prompt_cache_breakpoint_for_wire(marker, drop_params) == marker + + +def test_prompt_cache_breakpoint_for_wire_strips_an_unknown_key_under_drop_params() -> None: + assert prompt_cache_breakpoint_for_wire({"mode": "explicit", "note": "kept"}, True) == {"mode": "explicit"} From d004f519bb4d6e439d4b1dec79aefcf007f7cf9f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:06:48 -0700 Subject: [PATCH 3/7] fix(responses): read a non-flag deployment drop_params as unset in the marker walk, like the param mapper --- litellm/llms/openai/responses/transformation.py | 4 ++-- .../responses/test_openai_responses_transformation.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b80c82d26f4..224a520e514 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -10,7 +10,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, @@ -356,7 +356,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def _input_with_wire_safe_prompt_cache_breakpoints( self, input: str | ResponseInputParam, litellm_params: GenericLiteLLMParams ) -> str | ResponseInputParam: - deployment_drop_params: Final[object] = getattr(litellm_params, "drop_params", None) + deployment_drop_params: Final = normalize_drop_params(getattr(litellm_params, "drop_params", None)) if not (deployment_drop_params or litellm.drop_params) or not isinstance(input, list): return input wire_safe_items: Final = [self._item_with_wire_safe_prompt_cache_breakpoints(item) for item in input] diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 509cfec5075..6a0b96f5d28 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -3516,6 +3516,15 @@ def test_malformed_prompt_cache_breakpoint_is_forwarded_verbatim_without_drop_pa assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=drop_params)) == input +@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS) +def test_a_non_flag_deployment_drop_params_keeps_the_marker_like_the_param_mapper_does( + monkeypatch: pytest.MonkeyPatch, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", marker) + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params="maybe")) == input + + @pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) @pytest.mark.parametrize("marker", _VALID_MARKERS, ids=_VALID_MARKER_IDS) def test_valid_prompt_cache_breakpoint_reaches_the_wire_under_both_settings( From 69ebd0e8f2de4a8a9c1fbaac323f645b0286e992 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:34:16 -0700 Subject: [PATCH 4/7] fix(responses): drop every prompt_cache_breakpoint under drop_params when the model has no breakpoints Also drop a marker set on the message item itself, sanitize before the cache_control conversion so a client typo no longer erases that breakpoint, and skip items that carry no marker so unmarked history costs no extra walk --- .../llms/openai/responses/transformation.py | 105 ++++++++++++------ .../test_openai_responses_transformation.py | 59 ++++++++++ 2 files changed, 130 insertions(+), 34 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 04b49c48929..fa9bbbd7c21 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + supports_openai_prompt_cache_breakpoint, + targets_openai_api, +) from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -195,29 +199,54 @@ def _with_breakpoint_on_last_reading_part( return (*parts[:last], marked_last, *parts[last + 1 :]) -def _item_with_wire_safe_prompt_cache_breakpoints(item: object) -> object: - try: - fields: Final = _INPUT_ITEM.validate_python(item) - except ValidationError: +def _json_object_or_none(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None + + +def _json_array_or_empty(value: object) -> Sequence[object]: + return value if isinstance(value, (list, tuple)) else () + + +def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(item) + if fields is None or not _carries_prompt_cache_breakpoint(fields): return item - return {key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value) for key, value in fields.items()} + return { + key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers) + for key, value in fields.items() + if key != "prompt_cache_breakpoint" + } -def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object) -> object: - if key not in _INPUT_PART_LIST_KEYS: +def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool: + return "prompt_cache_breakpoint" in fields or any( + _parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS + ) + + +def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool: + return any(_has_own_prompt_cache_breakpoint(part) for part in _json_array_or_empty(parts)) + + +def _has_own_prompt_cache_breakpoint(part: object) -> bool: + fields: Final = _json_object_or_none(part) + return fields is not None and "prompt_cache_breakpoint" in fields + + +def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object: + if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value): return value - try: - parts: Final = _CONTENT_PARTS.validate_python(value) - except ValidationError: - return value - return [_part_with_wire_safe_prompt_cache_breakpoint(part) for part in parts] + return [_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in _json_array_or_empty(value)] -def _part_with_wire_safe_prompt_cache_breakpoint(part: Mapping[str, object]) -> Mapping[str, object]: - if "prompt_cache_breakpoint" not in part: +def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(part) + if fields is None or "prompt_cache_breakpoint" not in fields: return part - marker: Final = prompt_cache_breakpoint_for_wire(part["prompt_cache_breakpoint"], drop_params=True) - unmarked: Final = {key: value for key, value in part.items() if key != "prompt_cache_breakpoint"} + unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"} + if not keeps_markers: + return unmarked + marker: Final = prompt_cache_breakpoint_for_wire(fields["prompt_cache_breakpoint"], drop_params=True) return with_prompt_cache_breakpoint(unmarked, marker) @@ -542,12 +571,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): prompt_cache_options: object = None, ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]: validated_input: Final = self._validate_input_param(self._drop_bridge_minted_reasoning_items(input)) + marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints( + model=model, input=validated_input, litellm_params=litellm_params + ) marked_input: Final = ( - self._input_with_prompt_cache_breakpoints(validated_input) + self._input_with_prompt_cache_breakpoints(marker_safe_input) if self._reads_prompt_cache_breakpoints( model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options ) - else validated_input + else marker_safe_input ) maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any( is_anthropic_tool_search_chat_tool(tool) for tool in tools or () @@ -565,24 +597,28 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( model=model, input=replayable_input, tools=mapped_tools ) - marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints( - input=stripped_input, litellm_params=litellm_params - ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params ) - return self._drop_foreign_tool_call_item_ids(marker_safe_input), sanitized_tools + return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools - @staticmethod def _input_with_wire_safe_prompt_cache_breakpoints( - input: str | ResponseInputParam, litellm_params: GenericLiteLLMParams + self, model: str, input: str | ResponseInputParam, litellm_params: GenericLiteLLMParams ) -> str | ResponseInputParam: if not isinstance(input, list) or not _drops_params(litellm_params): return input - wire_safe_items: Final = [_item_with_wire_safe_prompt_cache_breakpoints(item) for item in input] + keeps_markers: Final = self._supports_prompt_cache_breakpoints(model=model, litellm_params=litellm_params) + wire_safe_items: Final = [_item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input] return cast("ResponseInputParam", wire_safe_items) # cast-ok: items keep their shape, minus a rejected marker + def _supports_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool: + provider: Final = _provider_name(self.custom_llm_provider) + base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) + return supports_openai_prompt_cache_breakpoint(model, provider) or ( + base_model != model and supports_openai_prompt_cache_breakpoint(base_model, provider) + ) + def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool: host: Final = _DeploymentHost.model_validate(litellm_params) return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url) @@ -698,14 +734,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _without_foreign_tool_call_item_id(item: object) -> object: - if not isinstance(item, dict): + fields: Final = _json_object_or_none(item) + if fields is None: return item - item_type: Final = item.get("type") - item_id: Final = item.get("id") + item_type: Final = fields.get("type") + item_id: Final = fields.get("id") genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix): return item - return {key: value for key, value in item.items() if key != "id"} + return {key: value for key, value in fields.items() if key != "id"} def _sanitized_tool_schemas_for_openai( self, @@ -728,7 +765,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): """ if tools is None or self.custom_llm_provider not in _PROVIDERS_WITH_OPENAI_SCHEMA_VALIDATOR: return tools - gate_model: Final = self._combinator_gate_model(model=model, litellm_params=litellm_params) + gate_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) sanitize: Final = ( flatten_combinators_and_drop_non_python_regex_patterns if self._rejects_top_level_schema_combinators(gate_model) @@ -744,9 +781,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return base_model.startswith(_MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS) @staticmethod - def _combinator_gate_model(model: str, litellm_params: GenericLiteLLMParams) -> str: - model_info: Final[object] = getattr(litellm_params, "model_info", None) - base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None + def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + model_info: Final = _json_object_or_none(getattr(litellm_params, "model_info", None)) + base_model: Final = None if model_info is None else model_info.get("base_model") return base_model if isinstance(base_model, str) and base_model else model @staticmethod diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 51336aaa50e..c3f5f985d80 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -4271,6 +4271,65 @@ def test_input_without_a_marker_is_untouched_under_drop_params( assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) == input +def _transformed_input_for(model: str, input: list[dict[str, object]], litellm_params: GenericLiteLLMParams) -> object: + request: Final = _CONFIGS[0].transform_responses_api_request( + model=model, + input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items + response_api_optional_request_params={}, + litellm_params=litellm_params, + headers={}, + ) + return request["input"] + + +@pytest.mark.parametrize( + "marker", (*_VALID_MARKERS, *_MALFORMED_MARKERS), ids=(*_VALID_MARKER_IDS, *_MALFORMED_MARKER_IDS) +) +def test_every_prompt_cache_breakpoint_is_dropped_under_drop_params_for_a_model_without_breakpoints( + monkeypatch: pytest.MonkeyPatch, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", marker) + dropped: Final = _transformed_input_for("local-model", input, GenericLiteLLMParams(drop_params=True)) + assert _single_part(dropped) == _PARTS["input_text"] + assert _transformed_input_for("local-model", input, GenericLiteLLMParams(drop_params=False)) == input + + +def test_a_declared_base_model_with_breakpoints_keeps_a_valid_marker_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", {"mode": "explicit"}) + litellm_params: Final = GenericLiteLLMParams(drop_params=True, model_info={"base_model": "gpt-6.1-sol"}) + assert _transformed_input_for("prod-deployment", input, litellm_params) == input + + +@pytest.mark.parametrize("content", ("Say ok", [{"type": "input_text", "text": "Say ok"}]), ids=("string", "parts")) +@pytest.mark.parametrize("marker", (_VALID_MARKERS[0], "yes"), ids=("explicit", "string")) +def test_a_prompt_cache_breakpoint_on_the_message_item_is_dropped_under_drop_params( + monkeypatch: pytest.MonkeyPatch, content: object, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + unmarked: Final[dict[str, object]] = {"type": "message", "role": "user", "content": content} + input: Final = [{**unmarked, "prompt_cache_breakpoint": marker}] + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) == [unmarked] + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=False)) == input + + +def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + part: Final = {"type": "input_text", "text": "Say ok", "cache_control": {"type": "ephemeral"}} + input: Final = [{"type": "message", "role": "user", "content": [{**part, "prompt_cache_breakpoint": "yes"}]}] + transformed: Final = _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) + assert _single_part(transformed) == { + "type": "input_text", + "text": "Say ok", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + + @pytest.mark.asyncio @pytest.mark.usefixtures(httpx_transport.__name__) async def test_aresponses_forwards_previous_response_id_to_openai() -> None: From dfe139eebb57d1b624a502579151ef2ebd798bb7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:31:23 -0700 Subject: [PATCH 5/7] fix(responses): keep a valid prompt_cache_breakpoint unless the model is known to reject it A deployment alias the cost map does not know (an Azure deployment name, a fine-tune, a litellm_proxy alias) kept its valid markers before this PR and must keep them under drop_params. base_model is read from litellm_params first, then model_info, as chat completions does. --- .../anthropic_cache_control_hook.py | 6 ++- .../llms/openai/responses/transformation.py | 25 +++++++---- .../test_openai_responses_transformation.py | 43 +++++++++++++++---- 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index b0deec24c1a..d02acf082f6 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -125,6 +125,10 @@ def _served_provider_for_model(model: str) -> str | None: def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True + + +def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None: hosted_flag: Final = ( None if custom_llm_provider is None @@ -137,7 +141,7 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str return model_map_flag version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) if version_match is None: - return False + return None version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index fa9bbbd7c21..efc0e694948 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -12,7 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, - supports_openai_prompt_cache_breakpoint, + openai_prompt_cache_breakpoint_support, targets_openai_api, ) from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers @@ -76,6 +76,13 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]]) +class _DeclaredModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + base_model: object = None + model_info: object = None + + class _DeploymentHost(BaseModel): """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`.""" @@ -608,16 +615,17 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> str | ResponseInputParam: if not isinstance(input, list) or not _drops_params(litellm_params): return input - keeps_markers: Final = self._supports_prompt_cache_breakpoints(model=model, litellm_params=litellm_params) + keeps_markers: Final = self._keeps_prompt_cache_breakpoints(model=model, litellm_params=litellm_params) wire_safe_items: Final = [_item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input] return cast("ResponseInputParam", wire_safe_items) # cast-ok: items keep their shape, minus a rejected marker - def _supports_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool: + def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool: provider: Final = _provider_name(self.custom_llm_provider) base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) - return supports_openai_prompt_cache_breakpoint(model, provider) or ( - base_model != model and supports_openai_prompt_cache_breakpoint(base_model, provider) + known_support: Final = ( + openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model)) ) + return next((support for support in known_support if support is not None), None) is not False def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool: host: Final = _DeploymentHost.model_validate(litellm_params) @@ -782,9 +790,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str: - model_info: Final = _json_object_or_none(getattr(litellm_params, "model_info", None)) - base_model: Final = None if model_info is None else model_info.get("base_model") - return base_model if isinstance(base_model, str) and base_model else model + declared: Final = _DeclaredModel.model_validate(litellm_params) + model_info: Final = _json_object_or_none(declared.model_info) + candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model")) + return next((name for name in candidates if isinstance(name, str) and name), model) @staticmethod def _sanitized_tool_entry( diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index c3f5f985d80..11f6c2742ba 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -4271,8 +4271,13 @@ def test_input_without_a_marker_is_untouched_under_drop_params( assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) == input -def _transformed_input_for(model: str, input: list[dict[str, object]], litellm_params: GenericLiteLLMParams) -> object: - request: Final = _CONFIGS[0].transform_responses_api_request( +def _transformed_input_for( + model: str, + input: list[dict[str, object]], + litellm_params: GenericLiteLLMParams, + config: BaseResponsesAPIConfig = _CONFIGS[0], +) -> object: + request: Final = config.transform_responses_api_request( model=model, input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items response_api_optional_request_params={}, @@ -4290,18 +4295,40 @@ def test_every_prompt_cache_breakpoint_is_dropped_under_drop_params_for_a_model_ ) -> None: monkeypatch.setattr(litellm, "drop_params", False) input: Final = _marked_message("input_text", marker) - dropped: Final = _transformed_input_for("local-model", input, GenericLiteLLMParams(drop_params=True)) + dropped: Final = _transformed_input_for("gpt-5.4-mini", input, GenericLiteLLMParams(drop_params=True)) assert _single_part(dropped) == _PARTS["input_text"] - assert _transformed_input_for("local-model", input, GenericLiteLLMParams(drop_params=False)) == input + assert _transformed_input_for("gpt-5.4-mini", input, GenericLiteLLMParams(drop_params=False)) == input -def test_a_declared_base_model_with_breakpoints_keeps_a_valid_marker_under_drop_params( - monkeypatch: pytest.MonkeyPatch, +@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS) +def test_a_valid_marker_survives_drop_params_on_a_deployment_name_with_unknown_breakpoint_support( + monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + litellm_params: Final = GenericLiteLLMParams(drop_params=True) + valid: Final = _marked_message("input_text", {"mode": "explicit"}) + malformed: Final = _marked_message("input_text", {"mode": "explicit", "ttl": "1h"}) + assert _transformed_input_for("litellm-qa-sol-alias", valid, litellm_params, config) == valid + dropped: Final = _transformed_input_for("litellm-qa-sol-alias", malformed, litellm_params, config) + assert _single_part(dropped) == _PARTS["input_text"] + + +def _declaring_base_model(where: str, base_model: str) -> GenericLiteLLMParams: + if where == "litellm_params": + return GenericLiteLLMParams(drop_params=True, base_model=base_model) + return GenericLiteLLMParams(drop_params=True, model_info={"base_model": base_model}) + + +@pytest.mark.parametrize("where", ("litellm_params", "model_info")) +def test_a_declared_base_model_decides_whether_a_valid_marker_survives_drop_params( + monkeypatch: pytest.MonkeyPatch, where: str ) -> None: monkeypatch.setattr(litellm, "drop_params", False) input: Final = _marked_message("input_text", {"mode": "explicit"}) - litellm_params: Final = GenericLiteLLMParams(drop_params=True, model_info={"base_model": "gpt-6.1-sol"}) - assert _transformed_input_for("prod-deployment", input, litellm_params) == input + kept: Final = _transformed_input_for("gpt-5.4-mini-prod", input, _declaring_base_model(where, "gpt-6.1-sol")) + assert kept == input + dropped: Final = _transformed_input_for("prod-deployment", input, _declaring_base_model(where, "gpt-5.4-mini")) + assert _single_part(dropped) == _PARTS["input_text"] @pytest.mark.parametrize("content", ("Say ok", [{"type": "input_text", "text": "Say ok"}]), ids=("string", "parts")) From f624b9b761c2c837e9bd0a1b888e80b79d420ac8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:04:35 -0700 Subject: [PATCH 6/7] fix(responses): drop a valid marker on known non-breakpoint models and keep the tool schema gate unchanged --- .../anthropic_cache_control_hook.py | 20 ++++++++++++--- .../llms/openai/responses/transformation.py | 17 ++++++++++--- .../test_openai_responses_transformation.py | 25 +++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index d02acf082f6..f2af9286c5f 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -13,7 +13,7 @@ import copy import itertools import os import re -from collections.abc import Callable, Container, Iterable, Mapping, Sequence +from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlparse @@ -72,6 +72,7 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") +_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) @@ -139,18 +140,29 @@ def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) if model_map_flag is not None: return model_map_flag + fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1]) + if fine_tuned_base is not None: + return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None + return _named_model_breakpoint_support(model) + + +def _named_model_breakpoint_support(model: str) -> bool | None: version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) if version_match is None: - return None + return False if next(_model_map_entries(model), None) is not None else None version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION -def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: +def _model_map_entries(model: str) -> Iterator[Mapping[str, object]]: import litellm entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1])) - flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict)) + return (entry for entry in entries if isinstance(entry, dict)) + + +def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: + flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in _model_map_entries(model)) return next((flag is True for flag in flags if flag is not None), None) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index efc0e694948..e56d54cda22 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints @@ -225,6 +225,11 @@ def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: b } +def _input_carries_marker(items: Iterable[object]) -> bool: + item_fields: Final = (_json_object_or_none(item) for item in items) + return any(_carries_prompt_cache_breakpoint(fields) for fields in item_fields if fields is not None) + + def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool: return "prompt_cache_breakpoint" in fields or any( _parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS @@ -613,7 +618,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): def _input_with_wire_safe_prompt_cache_breakpoints( self, model: str, input: str | ResponseInputParam, litellm_params: GenericLiteLLMParams ) -> str | ResponseInputParam: - if not isinstance(input, list) or not _drops_params(litellm_params): + if not isinstance(input, list) or not _drops_params(litellm_params) or not _input_carries_marker(input): return input keeps_markers: Final = self._keeps_prompt_cache_breakpoints(model=model, litellm_params=litellm_params) wire_safe_items: Final = [_item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input] @@ -773,7 +778,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): """ if tools is None or self.custom_llm_provider not in _PROVIDERS_WITH_OPENAI_SCHEMA_VALIDATOR: return tools - gate_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) + gate_model: Final = self._combinator_gate_model(model=model, litellm_params=litellm_params) sanitize: Final = ( flatten_combinators_and_drop_non_python_regex_patterns if self._rejects_top_level_schema_combinators(gate_model) @@ -788,6 +793,12 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): base_model: Final = bare_model.split(":")[1] if bare_model.startswith("ft:") else bare_model return base_model.startswith(_MODEL_FAMILIES_REJECTING_TOP_LEVEL_SCHEMA_COMBINATORS) + @staticmethod + def _combinator_gate_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + model_info: Final[object] = getattr(litellm_params, "model_info", None) + base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None + return base_model if isinstance(base_model, str) and base_model else model + @staticmethod def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str: declared: Final = _DeclaredModel.model_validate(litellm_params) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 11f6c2742ba..1b04933e861 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -4300,6 +4300,31 @@ def test_every_prompt_cache_breakpoint_is_dropped_under_drop_params_for_a_model_ assert _transformed_input_for("gpt-5.4-mini", input, GenericLiteLLMParams(drop_params=False)) == input +def _openai_model_the_cost_map_knows_without_a_breakpoint_flag() -> str: + entries: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python(litellm.model_cost) + return next( + name + for name, entry in entries.items() + if entry.get("litellm_provider") == "openai" + and "supports_prompt_cache_breakpoint" not in entry + and not name.startswith(("gpt-", "ft:")) + ) + + +@pytest.mark.parametrize( + "known_model", + (_openai_model_the_cost_map_knows_without_a_breakpoint_flag, lambda: "ft:gpt-4o-mini-2024-07-18:my-org::abc123"), + ids=("cost-map-model-without-flag", "fine-tune-of-older-gpt"), +) +def test_a_valid_marker_is_dropped_under_drop_params_on_a_model_known_to_lack_breakpoints( + monkeypatch: pytest.MonkeyPatch, known_model: Callable[[], str] +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", {"mode": "explicit"}) + dropped: Final = _transformed_input_for(known_model(), input, GenericLiteLLMParams(drop_params=True)) + assert _single_part(dropped) == _PARTS["input_text"] + + @pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS) def test_a_valid_marker_survives_drop_params_on_a_deployment_name_with_unknown_breakpoint_support( monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig From b7aee0ae73516fe4504978cb42f54d1cf978bdce Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:55:04 -0700 Subject: [PATCH 7/7] fix(responses): judge breakpoint support by model name so cost-map aliases keep valid markers --- .../anthropic_cache_control_hook.py | 18 +++++------ .../test_openai_responses_transformation.py | 32 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index f2af9286c5f..fc1f1830129 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -13,7 +13,7 @@ import copy import itertools import os import re -from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence +from collections.abc import Callable, Container, Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlparse @@ -73,6 +73,7 @@ CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") _FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)") +_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) @@ -147,22 +148,21 @@ def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str def _named_model_breakpoint_support(model: str) -> bool | None: - version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) + bare_model: Final = model.rsplit("/", 1)[-1].lower() + if _O_SERIES_PATTERN.match(bare_model) is not None: + return False + version_match: Final = _GPT_VERSION_PATTERN.match(bare_model) if version_match is None: - return False if next(_model_map_entries(model), None) is not None else None + return None version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION -def _model_map_entries(model: str) -> Iterator[Mapping[str, object]]: +def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: import litellm entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1])) - return (entry for entry in entries if isinstance(entry, dict)) - - -def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: - flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in _model_map_entries(model)) + flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict)) return next((flag is True for flag in flags if flag is not None), None) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 1b04933e861..8d51a6c8639 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -4300,31 +4300,31 @@ def test_every_prompt_cache_breakpoint_is_dropped_under_drop_params_for_a_model_ assert _transformed_input_for("gpt-5.4-mini", input, GenericLiteLLMParams(drop_params=False)) == input -def _openai_model_the_cost_map_knows_without_a_breakpoint_flag() -> str: - entries: Final = TypeAdapter(dict[str, dict[str, object]]).validate_python(litellm.model_cost) - return next( - name - for name, entry in entries.items() - if entry.get("litellm_provider") == "openai" - and "supports_prompt_cache_breakpoint" not in entry - and not name.startswith(("gpt-", "ft:")) - ) - - @pytest.mark.parametrize( - "known_model", - (_openai_model_the_cost_map_knows_without_a_breakpoint_flag, lambda: "ft:gpt-4o-mini-2024-07-18:my-org::abc123"), - ids=("cost-map-model-without-flag", "fine-tune-of-older-gpt"), + "model", + ("o4-mini", "ft:gpt-4o-mini-2024-07-18:my-org::abc123", "ft:o4-mini-2025-04-16:my-org::abc123"), + ids=("o-series", "fine-tune-of-older-gpt", "fine-tune-of-o-series"), ) def test_a_valid_marker_is_dropped_under_drop_params_on_a_model_known_to_lack_breakpoints( - monkeypatch: pytest.MonkeyPatch, known_model: Callable[[], str] + monkeypatch: pytest.MonkeyPatch, model: str ) -> None: monkeypatch.setattr(litellm, "drop_params", False) input: Final = _marked_message("input_text", {"mode": "explicit"}) - dropped: Final = _transformed_input_for(known_model(), input, GenericLiteLLMParams(drop_params=True)) + dropped: Final = _transformed_input_for(model, input, GenericLiteLLMParams(drop_params=True)) assert _single_part(dropped) == _PARTS["input_text"] +@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS) +def test_a_cost_map_entry_registered_under_the_deployment_name_keeps_a_valid_marker_under_drop_params( + monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + registered: Final = {"id": "litellm-qa-sol-alias", "supports_vision": True, "input_cost_per_token": 1e-6} + monkeypatch.setitem(litellm.model_cost, "litellm-qa-sol-alias", registered) + valid: Final = _marked_message("input_text", {"mode": "explicit"}) + assert _transformed_input_for("litellm-qa-sol-alias", valid, GenericLiteLLMParams(drop_params=True), config) == valid + + @pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS) def test_a_valid_marker_survives_drop_params_on_a_deployment_name_with_unknown_breakpoint_support( monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig