From b93d26bc854c0519d60d8fb1643689cbb36589b8 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 03:37:54 -0700 Subject: [PATCH] fix(responses): drop a malformed prompt_cache_breakpoint under drop_params on the direct /v1/responses path (#45330) * fix(responses): drop a malformed prompt_cache_breakpoint under drop_params on the direct /v1/responses path * refactor(responses): reuse the shared prompt_cache_breakpoint_for_wire helper in the chat bridge * fix(responses): read a non-flag deployment drop_params as unset in the marker walk, like the param mapper * fix(responses): drop every prompt_cache_breakpoint under drop_params when the model has no breakpoints Also drop a marker set on the message item itself, sanitize before the cache_control conversion so a client typo no longer erases that breakpoint, and skip items that carry no marker so unmarked history costs no extra walk * fix(responses): keep a valid prompt_cache_breakpoint unless the model is known to reject it A deployment alias the cost map does not know (an Azure deployment name, a fine-tune, a litellm_proxy alias) kept its valid markers before this PR and must keep them under drop_params. base_model is read from litellm_params first, then model_info, as chat completions does. * fix(responses): drop a valid marker on known non-breakpoint models and keep the tool schema gate unchanged * fix(responses): judge breakpoint support by model name so cost-map aliases keep valid markers * fix(responses): keep the caller's request stripped of cache_control as before when a marker is copied * fix(responses): keep valid prompt cache markers in place and let cache_control fill a dropped one Under drop_params the marker step now hands back the caller's own objects when every marker is already wire-safe, so a request with valid markers is no longer copied and walked a second time. A deployment that turns cache_control into markers keeps the caller's own markers too, whatever its base_model says, so one request never gets two answers about breakpoint support. On /v1/messages a malformed raw marker next to cache_control is dropped before the conversion, so the block gets the cache hint cache_control alone would give, as on /v1/responses. * test(responses): keep a null marker and the original key order on a model that reads breakpoints --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../transformation.py | 25 +- .../anthropic_cache_control_hook.py | 20 +- .../prompt_templates/common_utils.py | 15 +- .../messages/responses_adapters/handler.py | 24 +- .../llms/openai/responses/transformation.py | 164 ++++++++-- ...ses_bridge_prompt_cache_breakpoint_wire.py | 44 ++- ...ore_utils_prompt_templates_common_utils.py | 47 +++ .../test_responses_adapters_handler.py | 22 ++ .../test_openai_responses_transformation.py | 300 ++++++++++++++++++ 9 files changed, 605 insertions(+), 56 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 6173794a210..580e55273f6 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -36,6 +36,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( + prompt_cache_breakpoint_for_wire, responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) @@ -61,7 +62,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, - PromptCacheBreakpoint, Reasoning, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, @@ -631,19 +631,6 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li return "length" -_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) - - -def _prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: - if marker is None or not drop_params: - return marker - try: - return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) - except ValidationError: - verbose_logger.debug("Chat provider: dropping malformed prompt_cache_breakpoint %r under drop_params", marker) - return None - - def _input_file_from_file_value(file_value: object) -> dict[str, object]: if not isinstance(file_value, dict): return {"type": "input_file"} @@ -1679,7 +1666,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if original_type == "text": converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(item.get("text", ""), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: text -> %s", converted) @@ -1692,14 +1679,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(ChatCompletionImageObject, item), role ), ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) elif original_type == "input_audio": converted = with_prompt_cache_breakpoint( {"type": "input_audio", "input_audio": item.get("input_audio")}, - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: input_audio -> %s", converted) @@ -1715,7 +1702,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): _input_file_from_file_value( cast("ChatCompletionFileObject", item).get("file"), # cast-ok: type tag checked ), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: file -> %s", converted) @@ -1739,7 +1726,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Default to input_text for unknown types converted = with_prompt_cache_breakpoint( self._convert_content_str_to_input_text(str(item.get("text", item)), role), - _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), ) result.append(converted) verbose_logger.debug("Chat provider: unknown(%s) -> %s", original_type, converted) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index b0deec24c1a..fc1f1830129 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -72,6 +72,8 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint") OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6) _GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?") +_FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)") +_O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) @@ -125,6 +127,10 @@ def _served_provider_for_model(model: str) -> str | None: def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: + return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True + + +def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None: hosted_flag: Final = ( None if custom_llm_provider is None @@ -135,9 +141,19 @@ def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) if model_map_flag is not None: return model_map_flag - version_match: Final = _GPT_VERSION_PATTERN.match(model.rsplit("/", 1)[-1].lower()) - if version_match is None: + fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1]) + if fine_tuned_base is not None: + return False if _named_model_breakpoint_support(fine_tuned_base.group(1)) is False else None + return _named_model_breakpoint_support(model) + + +def _named_model_breakpoint_support(model: str) -> bool | None: + bare_model: Final = model.rsplit("/", 1)[-1].lower() + if _O_SERIES_PATTERN.match(bare_model) is not None: return False + version_match: Final = _GPT_VERSION_PATTERN.match(bare_model) + if version_match is None: + return None version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0)) return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 101f069c0f8..9bc53791b36 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -17,7 +17,7 @@ from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm from litellm import verbose_logger @@ -2073,6 +2073,19 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: return cast(_MarkedT, marked) # cast-ok: same block shape as the input plus the marker key +_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint) + + +def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object: + if marker is None or not drop_params: + return marker + try: + return _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + + def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: """Anthropic's ``cache_control`` mark rewritten as OpenAI's ``prompt_cache_breakpoint`` on the same block.""" if block.get("cache_control") is None: diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 99a07a1e9bf..9b22170c9db 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -15,7 +15,11 @@ from litellm.integrations.anthropic_cache_control_hook import ( supports_openai_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, +) from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -90,14 +94,28 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str return {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} +def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]: + if "prompt_cache_breakpoint" not in block: + return cache_control_as_prompt_cache_breakpoint(block) + wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True) + return cache_control_as_prompt_cache_breakpoint( + with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker) + ) + + def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, object]) -> BlockRewrite | None: """Client cache marks become breakpoints for a deployment that reads them, and a model that takes no breakpoints drops raw ones under drop_params.""" + drops_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params")) if _reads_prompt_cache_breakpoint(model, forwarded_kwargs): - return cache_control_as_prompt_cache_breakpoint + return ( + _cache_control_as_wire_safe_prompt_cache_breakpoint + if drops_params + else cache_control_as_prompt_cache_breakpoint + ) if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs): return None - if drops_unsupported_params(forwarded_kwargs.get("drop_params")): + if drops_params: return _without_prompt_cache_breakpoint return None diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index be40220f94d..903e6177e7b 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from functools import lru_cache from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints @@ -10,7 +10,11 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, targets_openai_api +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + openai_prompt_cache_breakpoint_support, + targets_openai_api, +) from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( @@ -20,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, drop_non_python_regex_patterns, flatten_combinators_and_drop_non_python_regex_patterns, + prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -66,6 +71,7 @@ _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS: Final = frozenset({LlmProviders.AZURE, _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING: Final = _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS _ROLES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"system", "developer", "user"}) _PART_TYPES_READING_PROMPT_CACHE_BREAKPOINTS: Final = frozenset({"input_text", "input_image", "input_file"}) +_INPUT_PART_LIST_KEYS: Final = frozenset({"content", "output"}) _COMPACT_TRACKING_PARAMS: Final = frozenset({"metadata", "user"}) _COMPACT_REJECTED_PARAMS: Final = frozenset( { @@ -93,6 +99,13 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]]) +class _DeclaredModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + base_model: object = None + model_info: object = None + + class _DeploymentHost(BaseModel): """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`.""" @@ -216,6 +229,63 @@ def _with_breakpoint_on_last_reading_part( return (*parts[:last], marked_last, *parts[last + 1 :]) +def _json_object_or_none(value: object) -> Mapping[str, object] | None: + return value if isinstance(value, Mapping) else None + + +def _json_array_or_empty(value: object) -> Sequence[object]: + return value if isinstance(value, (list, tuple)) else () + + +def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(item) + if fields is None or not _carries_prompt_cache_breakpoint(fields): + return item + wire_safe_fields: Final = { + key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers) + for key, value in fields.items() + if key != "prompt_cache_breakpoint" + } + unchanged: Final = "prompt_cache_breakpoint" not in fields and _same_objects( + wire_safe_fields.values(), fields.values() + ) + return item if unchanged else wire_safe_fields + + +def _same_objects(rebuilt: Iterable[object], originals: Iterable[object]) -> bool: + return all(new is old for new, old in zip(rebuilt, originals, strict=True)) + + +def _carries_prompt_cache_breakpoint(fields: Mapping[str, object]) -> bool: + return "prompt_cache_breakpoint" in fields or any( + _parts_carry_prompt_cache_breakpoint(fields.get(key)) for key in _INPUT_PART_LIST_KEYS + ) + + +def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool: + return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts)) + + +def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object: + if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value): + return value + parts: Final = _json_array_or_empty(value) + wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts) + return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts) + + +def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object: + fields: Final = _json_object_or_none(part) + if fields is None or "prompt_cache_breakpoint" not in fields: + return part + marker: Final = fields["prompt_cache_breakpoint"] + wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None + if keeps_markers and wire_marker == marker: + return part + unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, wire_marker) + + class _ReasoningSupportEntry(LiteLLMBaseModel): litellm_provider: str | None = None supports_reasoning: bool | None = None @@ -544,12 +614,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): validated_input: Final = self._validate_input_param( self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params) ) - marked_input: Final = ( - self._input_with_prompt_cache_breakpoints(validated_input) - if self._reads_prompt_cache_breakpoints( - model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options - ) - else validated_input + reads_breakpoints: Final = self._reads_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options + ) + marker_safe_input: Final = self._input_with_wire_safe_prompt_cache_breakpoints( + model=model, input=validated_input, litellm_params=litellm_params, reads_breakpoints=reads_breakpoints ) maps_anthropic_tool_search: Final = self._targets_openai_api(litellm_params) and any( is_anthropic_tool_search_chat_tool(tool) for tool in tools or () @@ -559,20 +628,71 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if maps_anthropic_tool_search else tools ) - replayable_input: Final = ( - input_without_unanswered_anthropic_tool_search_calls(marked_input) - if maps_anthropic_tool_search - else marked_input - ) stripped_input, stripped_tools = self.remove_cache_control_flag_from_input_and_tools( - model=model, input=replayable_input, tools=mapped_tools + model=model, + input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search), + tools=mapped_tools, ) + if marker_safe_input is not validated_input: + self._strip_caller_input_as_if_unmarked( + model=model, + caller_input=validated_input, + reads_breakpoints=reads_breakpoints, + maps_anthropic_tool_search=maps_anthropic_tool_search, + ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params ) return self._drop_foreign_tool_call_item_ids(stripped_input), sanitized_tools + def _replayable_input( + self, input: str | ResponseInputParam, reads_breakpoints: bool, maps_anthropic_tool_search: bool + ) -> str | ResponseInputParam: + marked_input: Final = self._input_with_prompt_cache_breakpoints(input) if reads_breakpoints else input + if not maps_anthropic_tool_search: + return marked_input + return input_without_unanswered_anthropic_tool_search_calls(marked_input) + + def _strip_caller_input_as_if_unmarked( + self, + model: str, + caller_input: str | ResponseInputParam, + reads_breakpoints: bool, + maps_anthropic_tool_search: bool, + ) -> None: + self.remove_cache_control_flag_from_input_and_tools( + model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search) + ) + + def _input_with_wire_safe_prompt_cache_breakpoints( + self, + model: str, + input: str | ResponseInputParam, + litellm_params: GenericLiteLLMParams, + reads_breakpoints: bool, + ) -> str | ResponseInputParam: + if not isinstance(input, list) or not drops_params(litellm_params): + return input + keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints( + model=model, litellm_params=litellm_params + ) + wire_safe_items: Final = tuple( + _item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input + ) + if _same_objects(wire_safe_items, input): + return input + wire_safe_input: Final = list(wire_safe_items) + return cast("ResponseInputParam", wire_safe_input) # cast-ok: items keep their shape, minus a rejected marker + + def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool: + provider: Final = _provider_name(self.custom_llm_provider) + base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params) + known_support: Final = ( + openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model)) + ) + return next((support for support in known_support if support is not None), None) is not False + def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool: host: Final = _DeploymentHost.model_validate(litellm_params) return self.custom_llm_provider == LlmProviders.OPENAI and targets_openai_api(host.api_base or host.base_url) @@ -688,14 +808,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _without_foreign_tool_call_item_id(item: object) -> object: - if not isinstance(item, dict): + fields: Final = _json_object_or_none(item) + if fields is None: return item - item_type: Final = item.get("type") - item_id: Final = item.get("id") + item_type: Final = fields.get("type") + item_id: Final = fields.get("id") genuine_prefix: Final = TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE.get(item_type) if isinstance(item_type, str) else None if genuine_prefix is None or not isinstance(item_id, str) or item_id.startswith(genuine_prefix): return item - return {key: value for key, value in item.items() if key != "id"} + return {key: value for key, value in fields.items() if key != "id"} def _sanitized_tool_schemas_for_openai( self, @@ -739,6 +860,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None return base_model if isinstance(base_model, str) and base_model else model + @staticmethod + def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + declared: Final = _DeclaredModel.model_validate(litellm_params) + model_info: Final = _json_object_or_none(declared.model_info) + candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model")) + return next((name for name in candidates if isinstance(name, str) and name), model) + @staticmethod def _sanitized_tool_entry( entry: Mapping[str, object], diff --git a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py index 12a7bfe68e9..33338cae95d 100644 --- a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py @@ -39,14 +39,6 @@ _CASES: Final[tuple[tuple[str, Mode, JsonValue, JsonValue], ...]] = ( ) _CASE_IDS: Final = tuple(case[0] for case in _CASES) _CASE_VALUES: Final = tuple(case[1:] for case in _CASES) -_ADAPTER_CASES: Final[tuple[tuple[str, Mode, JsonValue, JsonValue], ...]] = ( - ("valid-on", "on", pcb.EXPLICIT, pcb.EXPLICIT), - ("valid-off", "off", pcb.EXPLICIT, pcb.EXPLICIT), - ("malformed-on", "on", "yes", "yes"), - ("malformed-off", "off", "yes", "yes"), -) -_ADAPTER_CASE_IDS: Final = tuple(case[0] for case in _ADAPTER_CASES) -_ADAPTER_CASE_VALUES: Final = tuple(case[1:] for case in _ADAPTER_CASES) @dataclass(frozen=True, slots=True) @@ -331,7 +323,7 @@ def _anthropic_text(message: anthropic.types.Message, marker: str) -> None: assert content.type == "text" and content.text == rv.answer(marker), message -@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _ADAPTER_CASE_VALUES, ids=_ADAPTER_CASE_IDS) +@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS) def test_anthropic_sdk_marker_on_user_text( bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue ) -> None: @@ -540,14 +532,17 @@ def test_anthropic_sdk_cache_control_on_an_assistant_turn_is_dropped(bridge: _Br ], "OpenAI accepts a breakpoint on output_text but writes no cache for it (live gpt-5.6, 2026-10-12)" -def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> None: +@pytest.mark.parametrize( + ("mode", "expected"), (("on", pcb.EXPLICIT), ("off", _UNKNOWN_KEY)), ids=("normalized-on", "verbatim-off") +) +def test_native_responses_marker_with_an_unknown_key(bridge: _Bridge, mode: Mode, expected: JsonValue) -> None: marker: Final = uuid.uuid4().hex block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)} response: Final = bridge.gateway.request( "POST", "/v1/responses", { - "model": bridge.on, + "model": bridge.model(mode), "input": [{"type": "message", "role": "user", "content": [pcb.marked(block, _UNKNOWN_KEY)]}], **pcb.NO_CACHE, }, @@ -559,8 +554,31 @@ def test_native_responses_request_never_enters_the_bridge(bridge: _Bridge) -> No request: Final = pcb.posted(bridge.wire, marker) _wire_body(request) on_wire: Final = pcb.single_block(pcb.input_items(request), "user") - assert on_wire == pcb.marked(block, _UNKNOWN_KEY), on_wire - bridge.spend.landed(bridge.on, response.headers["x-litellm-call-id"], marker) + assert on_wire == pcb.marked(block, expected), on_wire + bridge.spend.landed(bridge.model(mode), response.headers["x-litellm-call-id"], marker) + + +@pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS) +def test_openai_sdk_native_responses_marker_on_user_text( + bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue +) -> None: + marker: Final = uuid.uuid4().hex + block: Final[dict[str, JsonValue]] = {"type": "input_text", "text": pcb.prompt(marker)} + with openai.OpenAI(api_key=bridge.gateway.key, base_url=_v1(bridge.gateway), max_retries=0) as client: + raw: Final = client.responses.with_raw_response.create( + model=bridge.model(mode), + input=[{"type": "message", "role": "user", "content": [pcb.marked(block, breakpoint)]}], + extra_body=dict(pcb.NO_CACHE), + ) + response: Final = raw.parse() + assert pcb.answers(response.id, marker), response + assert response.output_text == rv.answer(marker), response + request: Final = pcb.posted(bridge.wire, marker) + _wire_body(request) + on_wire: Final = pcb.single_block(pcb.input_items(request), "user") + assert on_wire["type"] == "input_text" and on_wire["text"] == pcb.prompt(marker), on_wire + pcb.assert_marker(on_wire, expected) + bridge.spend.landed(bridge.model(mode), raw.headers["x-litellm-call-id"], marker) @pytest.mark.parametrize("breakpoint", _MALFORMED_VALUES, ids=_MALFORMED_IDS) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index b9675f6fd5c..72e72714cd4 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -30,6 +30,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( merge_consecutive_system_messages, native_redacted_thinking_data, parse_tool_call_arguments, + prompt_cache_breakpoint_for_wire, responses_reasoning_items_from_thinking_blocks, salvage_concatenated_tool_arguments, schema_grammar_branch_count, @@ -2328,6 +2329,52 @@ def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_w assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) +_MALFORMED_PROMPT_CACHE_BREAKPOINTS: Final = ( + "yes", + 1, + ["explicit"], + "", + {}, + {"mode": "bogus"}, + {"mode": "explicit", "ttl": "1h"}, +) +_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS: Final = ( + "string", + "int", + "list", + "empty-string", + "empty-object", + "bogus-mode", + "bad-ttl", +) +_VALID_PROMPT_CACHE_BREAKPOINTS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"}) + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +def test_prompt_cache_breakpoint_for_wire_passes_a_missing_marker_through(drop_params: bool) -> None: + assert prompt_cache_breakpoint_for_wire(None, drop_params) is None + + +@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) +def test_prompt_cache_breakpoint_for_wire_drops_a_malformed_marker_under_drop_params(marker: object) -> None: + assert prompt_cache_breakpoint_for_wire(marker, True) is None + + +@pytest.mark.parametrize("marker", _MALFORMED_PROMPT_CACHE_BREAKPOINTS, ids=_MALFORMED_PROMPT_CACHE_BREAKPOINT_IDS) +def test_prompt_cache_breakpoint_for_wire_forwards_a_malformed_marker_without_drop_params(marker: object) -> None: + assert prompt_cache_breakpoint_for_wire(marker, False) is marker + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +@pytest.mark.parametrize("marker", _VALID_PROMPT_CACHE_BREAKPOINTS, ids=("explicit", "explicit-30m")) +def test_prompt_cache_breakpoint_for_wire_keeps_a_valid_marker(marker: dict[str, str], drop_params: bool) -> None: + assert prompt_cache_breakpoint_for_wire(marker, drop_params) == marker + + +def test_prompt_cache_breakpoint_for_wire_strips_an_unknown_key_under_drop_params() -> None: + assert prompt_cache_breakpoint_for_wire({"mode": "explicit", "note": "kept"}, True) == {"mode": "explicit"} + + def test_cache_control_is_replaced_by_an_explicit_prompt_cache_breakpoint_on_the_same_block() -> None: marked: Final = {"type": "text", "text": "rules", "cache_control": {"type": "ephemeral", "ttl": "1h"}} diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 7de722e1171..bba5413fbdf 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -556,6 +556,28 @@ def test_raw_prompt_cache_breakpoints_reach_a_deployment_unless_drop_params_drop ] +def _gpt_5_6_input_for_a_cached_block(drop_params: bool, raw_mark: Mapping[str, object]) -> object: + responses_kwargs: Final = _build_responses_kwargs( + model="openai/gpt-5.6", + extra_kwargs={**OPENAI_API, "drop_params": drop_params}, + max_tokens=1024, + messages=[{"role": "user", "content": [{"type": "text", "text": "look", "cache_control": EPHEMERAL, **raw_mark}]}], + ) + return responses_kwargs["input"] + + +def test_a_malformed_raw_marker_beside_cache_control_gives_way_to_it_only_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + malformed: Final = {"prompt_cache_breakpoint": "yes"} + + assert _gpt_5_6_input_for_a_cached_block(True, malformed) == _gpt_5_6_input_for_a_cached_block(True, {}) + assert _gpt_5_6_input_for_a_cached_block(False, malformed) == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "look", **malformed}]} + ] + + @pytest.mark.parametrize( ("model", "extra_kwargs"), [ diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 48bb0955f34..a185cdaf718 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -25,6 +25,7 @@ from litellm.types.llms.openai import ( OutputTextDeltaEvent, ResponseAPIUsage, ResponseCompletedEvent, + ResponseInputParam, ResponsesAPIResponse, ResponsesAPIStreamEvents, ) @@ -4246,6 +4247,305 @@ async def test_extra_body_merges_with_request_data(extra_body_mock_response_data assert request_body["custom_field"] == "custom_value" +_MALFORMED_MARKERS: Final = ("yes", 1, {"mode": "bogus"}, {"mode": "explicit", "ttl": "1h"}) +_MALFORMED_MARKER_IDS: Final = ("string", "int", "bogus-mode", "bad-ttl") +_VALID_MARKERS: Final = ({"mode": "explicit"}, {"mode": "explicit", "ttl": "30m"}) +_VALID_MARKER_IDS: Final = ("explicit", "explicit-30m") +_PARTS: Final[Mapping[str, Mapping[str, object]]] = { + "input_text": {"type": "input_text", "text": "Say ok"}, + "input_image": {"type": "input_image", "image_url": "https://example.com/a.png"}, + "input_file": {"type": "input_file", "file_id": "file-abc"}, +} +_CONFIGS: Final = (OpenAIResponsesAPIConfig(), AzureOpenAIResponsesAPIConfig()) +_CONFIG_IDS: Final = ("openai", "azure") + + +def _marked_message(part_type: str, marker: object) -> list[dict[str, object]]: + return [{"type": "message", "role": "user", "content": [{**_PARTS[part_type], "prompt_cache_breakpoint": marker}]}] + + +def _transformed_input( + config: BaseResponsesAPIConfig, input: str | list[dict[str, object]], litellm_params: GenericLiteLLMParams +) -> object: + request: Final = config.transform_responses_api_request( + model="gpt-6.1-sol", + input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items + response_api_optional_request_params={}, + litellm_params=litellm_params, + headers={}, + ) + return request["input"] + + +def _single_part(transformed_input: object) -> Mapping[str, object]: + (item,) = cast(list[Mapping[str, object]], transformed_input) + (part,) = cast(list[Mapping[str, object]], item["content"]) + return part + + +@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS) +@pytest.mark.parametrize("part_type", tuple(_PARTS)) +@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS) +def test_malformed_prompt_cache_breakpoint_is_dropped_under_deployment_drop_params( + monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig, part_type: str, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + transformed: Final = _transformed_input( + config, _marked_message(part_type, marker), GenericLiteLLMParams(drop_params=True) + ) + assert _single_part(transformed) == _PARTS[part_type] + + +@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS) +def test_malformed_prompt_cache_breakpoint_is_dropped_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + transformed: Final = _transformed_input(_CONFIGS[0], _marked_message("input_text", marker), GenericLiteLLMParams()) + assert _single_part(transformed) == _PARTS["input_text"] + + +@pytest.mark.parametrize("drop_params", (False, None), ids=("false", "null")) +@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS) +def test_malformed_prompt_cache_breakpoint_is_forwarded_verbatim_without_drop_params( + monkeypatch: pytest.MonkeyPatch, marker: object, drop_params: bool | None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", marker) + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=drop_params)) == input + + +@pytest.mark.parametrize("marker", _MALFORMED_MARKERS, ids=_MALFORMED_MARKER_IDS) +def test_a_non_flag_deployment_drop_params_keeps_the_marker_like_the_param_mapper_does( + monkeypatch: pytest.MonkeyPatch, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", marker) + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params="maybe")) == input + + +@pytest.mark.parametrize("drop_params", (True, False), ids=("drop", "keep")) +@pytest.mark.parametrize("marker", _VALID_MARKERS, ids=_VALID_MARKER_IDS) +def test_valid_prompt_cache_breakpoint_reaches_the_wire_under_both_settings( + monkeypatch: pytest.MonkeyPatch, marker: Mapping[str, str], drop_params: bool +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", marker) + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=drop_params)) == input + + +def test_prompt_cache_breakpoint_unknown_key_is_trimmed_under_drop_params(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", {"mode": "explicit", "note": "kept"}) + dropped: Final = _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) + assert _single_part(dropped)["prompt_cache_breakpoint"] == {"mode": "explicit"} + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=False)) == input + + +def test_malformed_prompt_cache_breakpoint_on_a_tool_output_part_is_dropped_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + marked_output: Final = [{"type": "input_text", "text": "42", "prompt_cache_breakpoint": "yes"}] + input: Final[list[dict[str, object]]] = [{"type": "function_call_output", "call_id": "call_1", "output": marked_output}] + (item,) = cast( + list[Mapping[str, object]], _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) + ) + assert item["output"] == [{"type": "input_text", "text": "42"}] + + +@pytest.mark.parametrize("input", ("hi", [{"type": "message", "role": "user", "content": "hi"}]), ids=("string", "unmarked")) +def test_input_without_a_marker_is_untouched_under_drop_params( + monkeypatch: pytest.MonkeyPatch, input: str | list[dict[str, object]] +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) == input + + +def _transformed_input_for( + model: str, + input: list[dict[str, object]], + litellm_params: GenericLiteLLMParams, + config: BaseResponsesAPIConfig = _CONFIGS[0], +) -> object: + request: Final = config.transform_responses_api_request( + model=model, + input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items + response_api_optional_request_params={}, + litellm_params=litellm_params, + headers={}, + ) + return request["input"] + + +@pytest.mark.parametrize( + "marker", (*_VALID_MARKERS, *_MALFORMED_MARKERS), ids=(*_VALID_MARKER_IDS, *_MALFORMED_MARKER_IDS) +) +def test_every_prompt_cache_breakpoint_is_dropped_under_drop_params_for_a_model_without_breakpoints( + monkeypatch: pytest.MonkeyPatch, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", marker) + dropped: Final = _transformed_input_for("gpt-5.4-mini", input, GenericLiteLLMParams(drop_params=True)) + assert _single_part(dropped) == _PARTS["input_text"] + assert _transformed_input_for("gpt-5.4-mini", input, GenericLiteLLMParams(drop_params=False)) == input + + +@pytest.mark.parametrize( + "model", + ("o4-mini", "ft:gpt-4o-mini-2024-07-18:my-org::abc123", "ft:o4-mini-2025-04-16:my-org::abc123"), + ids=("o-series", "fine-tune-of-older-gpt", "fine-tune-of-o-series"), +) +def test_a_valid_marker_is_dropped_under_drop_params_on_a_model_known_to_lack_breakpoints( + monkeypatch: pytest.MonkeyPatch, model: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", {"mode": "explicit"}) + dropped: Final = _transformed_input_for(model, input, GenericLiteLLMParams(drop_params=True)) + assert _single_part(dropped) == _PARTS["input_text"] + + +@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS) +def test_a_cost_map_entry_registered_under_the_deployment_name_keeps_a_valid_marker_under_drop_params( + monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + registered: Final = {"id": "litellm-qa-sol-alias", "supports_vision": True, "input_cost_per_token": 1e-6} + monkeypatch.setitem(litellm.model_cost, "litellm-qa-sol-alias", registered) + valid: Final = _marked_message("input_text", {"mode": "explicit"}) + assert _transformed_input_for("litellm-qa-sol-alias", valid, GenericLiteLLMParams(drop_params=True), config) == valid + + +@pytest.mark.parametrize("config", _CONFIGS, ids=_CONFIG_IDS) +def test_a_valid_marker_survives_drop_params_on_a_deployment_name_with_unknown_breakpoint_support( + monkeypatch: pytest.MonkeyPatch, config: BaseResponsesAPIConfig +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + litellm_params: Final = GenericLiteLLMParams(drop_params=True) + valid: Final = _marked_message("input_text", {"mode": "explicit"}) + malformed: Final = _marked_message("input_text", {"mode": "explicit", "ttl": "1h"}) + assert _transformed_input_for("litellm-qa-sol-alias", valid, litellm_params, config) == valid + dropped: Final = _transformed_input_for("litellm-qa-sol-alias", malformed, litellm_params, config) + assert _single_part(dropped) == _PARTS["input_text"] + + +def _declaring_base_model(where: str, base_model: str) -> GenericLiteLLMParams: + if where == "litellm_params": + return GenericLiteLLMParams(drop_params=True, base_model=base_model) + return GenericLiteLLMParams(drop_params=True, model_info={"base_model": base_model}) + + +@pytest.mark.parametrize("where", ("litellm_params", "model_info")) +def test_a_declared_base_model_decides_whether_a_valid_marker_survives_drop_params( + monkeypatch: pytest.MonkeyPatch, where: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + input: Final = _marked_message("input_text", {"mode": "explicit"}) + kept: Final = _transformed_input_for("gpt-5.4-mini-prod", input, _declaring_base_model(where, "gpt-6.1-sol")) + assert kept == input + dropped: Final = _transformed_input_for("prod-deployment", input, _declaring_base_model(where, "gpt-5.4-mini")) + assert _single_part(dropped) == _PARTS["input_text"] + + +def test_a_deployment_that_marks_cache_control_keeps_the_callers_own_marker_whatever_its_base_model_says( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + cached: Final = {"type": "input_text", "text": "a", "cache_control": {"type": "ephemeral"}} + marked: Final = {"type": "input_text", "text": "b", "prompt_cache_breakpoint": {"mode": "explicit"}} + input: Final = [{"type": "message", "role": "user", "content": [cached, marked]}] + transformed: Final = _transformed_input_for("gpt-6.1-sol", input, _declaring_base_model("model_info", "gpt-5.5")) + (item,) = cast(list[Mapping[str, object]], transformed) + parts: Final = cast(list[Mapping[str, object]], item["content"]) + assert [part.get("prompt_cache_breakpoint") for part in parts] == [{"mode": "explicit"}, {"mode": "explicit"}] + + +def test_a_model_that_takes_markers_sends_kept_parts_byte_for_byte_and_drops_only_the_rejected_marker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + kept: Final = [ + {"type": "input_text", "text": "a", "prompt_cache_breakpoint": None}, + {"prompt_cache_breakpoint": {"mode": "explicit"}, "type": "input_text", "text": "b"}, + ] + rejected: Final = {"type": "input_text", "text": "c"} + bad_ttl: Final = {"prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}} + input: Final = [ + {"type": "message", "role": "user", "content": kept}, + {"type": "message", "role": "user", "content": [{**bad_ttl, **rejected}]}, + ] + transformed: Final = _transformed_input_for("gpt-6.1-sol", input, GenericLiteLLMParams(drop_params=True)) + assert json.dumps(transformed) == json.dumps( + [{"type": "message", "role": "user", "content": kept}, {"type": "message", "role": "user", "content": [rejected]}] + ) + + +@pytest.mark.parametrize("content", ("Say ok", [{"type": "input_text", "text": "Say ok"}]), ids=("string", "parts")) +@pytest.mark.parametrize("marker", (_VALID_MARKERS[0], "yes"), ids=("explicit", "string")) +def test_a_prompt_cache_breakpoint_on_the_message_item_is_dropped_under_drop_params( + monkeypatch: pytest.MonkeyPatch, content: object, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + unmarked: Final[dict[str, object]] = {"type": "message", "role": "user", "content": content} + input: Final = [{**unmarked, "prompt_cache_breakpoint": marker}] + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) == [unmarked] + assert _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=False)) == input + + +def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + part: Final = {"type": "input_text", "text": "Say ok", "cache_control": {"type": "ephemeral"}} + input: Final = [{"type": "message", "role": "user", "content": [{**part, "prompt_cache_breakpoint": "yes"}]}] + transformed: Final = _transformed_input(_CONFIGS[0], input, GenericLiteLLMParams(drop_params=True)) + assert _single_part(transformed) == { + "type": "input_text", + "text": "Say ok", + "prompt_cache_breakpoint": {"mode": "explicit"}, + } + + +def _caller_input_after_transform(model: str, marker: object) -> str: + hint: Final = {"type": "ephemeral"} + marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker} + input: Final = [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "Say ok", "cache_control": hint, **marked}, + {"type": "input_text", "text": "Thanks", "cache_control": hint}, + ], + "cache_control": hint, + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}], + }, + ] + _CONFIGS[0].transform_responses_api_request( + model=model, + input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(drop_params=True), + headers={}, + ) + return json.dumps(input) + + +@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol")) +@pytest.mark.parametrize("marker", (_VALID_MARKERS[0], "yes"), ids=("explicit", "malformed")) +def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_unmarked_one_would( + monkeypatch: pytest.MonkeyPatch, model: str, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + marker_json: Final = f', "prompt_cache_breakpoint": {json.dumps(marker)}' + unmarked: Final = _caller_input_after_transform(model, None) + assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked + + @pytest.mark.asyncio @pytest.mark.usefixtures(httpx_transport.__name__) async def test_aresponses_forwards_previous_response_id_to_openai() -> None: