diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 62351d8e39a..3f3a8da6089 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -57,7 +57,7 @@ from litellm.types.utils import ( ) from litellm.utils import convert_to_model_response_object -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, without_cache_control if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -387,23 +387,19 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): messages: list[AllMessageValues], tools: list["ChatCompletionToolParam"] | None = None, ) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, - ) from litellm.types.llms.openai import ChatCompletionToolParam - for i, message in enumerate(messages): - messages[i] = cast( - AllMessageValues, - filter_value_from_dict(message, "cache_control"), - ) - if tools is not None: - for i, tool in enumerate(tools): - tools[i] = cast( - ChatCompletionToolParam, - filter_value_from_dict(tool, "cache_control"), - ) - return messages, tools + new_messages: Final = [ # mutable-ok: the declared return type is list[AllMessageValues] + cast(AllMessageValues, without_cache_control(message)) for message in messages + ] + new_tools: Final = ( + [ # mutable-ok: the declared return type is list[ChatCompletionToolParam] + cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools + ] + if tools is not None + else None + ) + return new_messages, new_tools def _targets_openai_hosted_endpoint( self, diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index b47edee9976..d9f86d8b6b0 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -2,6 +2,7 @@ Common helpers / utils across al OpenAI endpoints """ +import copy import hashlib import inspect import json @@ -36,6 +37,37 @@ from litellm.llms.custom_httpx.http_handler import ( ) +def _contains_cache_control(value: object) -> bool: + # Iterative rather than recursive: the code-quality gate bans unignored recursion here, + # and request payloads are attacker-shaped, so an explicit worklist has no stack depth to blow. + pending: list[object] = [value] # mutable-ok: a local worklist, never returned or stored + while pending: + node = pending.pop() # rebind-ok: the loop variable of an explicit worklist + if isinstance(node, dict): + if "cache_control" in node: + return True + pending.extend(node.values()) + elif isinstance(node, list): + pending.extend(node) + return False + + +def without_cache_control(item: object) -> object: + """Return `item` with every `cache_control` key removed, without touching the original. + + `filter_value_from_dict` deletes the key in place and recurses into nested dicts and + lists, so the caller's own message, tool or input object has to be copied first: it + may reuse the same list on a provider that does support prompt caching. The copy is + skipped when there is nothing to strip, which keeps the object identity the + pass-through paths rely on and avoids a deep copy on every request. + """ + from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict + + if not isinstance(item, dict) or not _contains_cache_control(item): + return item + return filter_value_from_dict(copy.deepcopy(item), "cache_control") + + def _get_client_init_params(cls: type) -> tuple[str, ...]: """Extract __init__ parameter names (excluding 'self') from a class.""" return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self") diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 6c1d8698652..1683433af5f 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -30,7 +30,7 @@ from litellm.types.responses.main import * from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, without_cache_control from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -371,24 +371,30 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): the chat path. Strips Anthropic-only `cache_control` markers from Responses API input content blocks and tools. - `filter_value_from_dict` mutates each dict in place, so the same - objects are returned. + `filter_value_from_dict` deletes the key in place and recurses, so each + item is copied first: the caller keeps its own input list and may reuse + it on a provider that does support prompt caching. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, + new_input: Final = ( + cast( # cast-ok: the comprehension rebuilds the input item for item + ResponseInputParam, + [ # mutable-ok: the declared return type is ResponseInputParam + without_cache_control(item) for item in input + ], + ) + if isinstance(input, list) + else input ) - if isinstance(input, list): - for item in input: - if isinstance(item, dict): - filter_value_from_dict(cast(dict, item), "cache_control") + new_tools: Final = ( + [ # mutable-ok: the declared return type is List[ALL_RESPONSES_API_TOOL_PARAMS] + without_cache_control(tool) for tool in tools + ] + if tools is not None + else None + ) - if tools is not None: - for tool in tools: - if isinstance(tool, dict): - filter_value_from_dict(cast(dict, tool), "cache_control") - - return input, tools + return new_input, new_tools def _drop_foreign_tool_call_item_ids(self, input: str | ResponseInputParam) -> str | ResponseInputParam: if self.custom_llm_provider not in _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS or not isinstance(input, list): diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 53c5b9d7cbc..4b7dc6868c3 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -2,6 +2,9 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ +import copy +import json + import pytest from typing import Final @@ -1406,3 +1409,107 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: ) assert request["tools"][0] is tool + + +class TestCacheControlStrippingDoesNotMutateCallerInput: + """ + Stripping cache_control for a provider that cannot use it must not reach back + into the caller's own message and tool objects. + + filter_value_from_dict deletes the key in place and recurses into nested dicts + and lists, and remove_cache_control_flag_from_messages_and_tools assigned the + result back into the caller's list, so one call to any OpenAI-compatible + provider permanently stripped cache_control from a list the caller still holds. + Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt + caching, with no error and full-price billing. + """ + + def setup_method(self): + self.config = OpenAIGPTConfig() + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _messages(): + return [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": "Hello", + "cache_control": {"type": "ephemeral"}, + }, + ] + + @staticmethod + def _tools(): + return [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + def _transform(self, messages, tools): + return self.config.transform_request( + model="gpt-4o", + messages=messages, + optional_params={"tools": tools}, + litellm_params={"custom_llm_provider": "openai", "api_base": None}, + headers={}, + ) + + def test_caller_messages_and_tools_keep_cache_control(self): + messages, tools = self._messages(), self._tools() + messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools) + + request = self._transform(messages, tools) + + # the outbound body must still be stripped, both message-level and nested + assert "cache_control" not in json.dumps(request) + # and the caller's objects must be untouched, nested content blocks included + assert messages == messages_before + assert tools == tools_before + + def test_a_later_anthropic_call_still_sees_cache_control(self): + """The user-visible consequence: the same message list reused on a provider + that does support caching must still carry the cache breakpoints.""" + messages = self._messages() + + self._transform(messages, self._tools()) + + anthropic_body = litellm.AnthropicConfig().transform_request( + model="claude-3-5-sonnet-20240620", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "cache_control" in json.dumps(anthropic_body) + + @staticmethod + def _artifact_tool(): + return { + "type": "function", + "function": { + "name": "Artifact", + "parameters": { + "type": "object", + "properties": {"field": {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN}}, + "required": ["field"], + }, + }, + } diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 0ef45501d91..f03bc15a3dc 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -202,6 +202,52 @@ class TestOpenAIResponsesAPIConfig: assert "cache_control" not in result["tools"][0] assert result["tools"][0]["name"] == "get_weather" + def test_transform_does_not_strip_cache_control_from_the_callers_input(self): + """Stripping for OpenAI must not reach back into the caller's own objects. + + `filter_value_from_dict` deletes the key in place and recurses, and + `_validate_input_param` passes plain dict items through by reference, so + the caller's input list used to lose its cache breakpoints. Reusing that + list on Anthropic or Bedrock afterwards then silently lost prompt caching. + Same defect as the Chat Completions path. + """ + input_with_cache_control = [ + { + "role": "system", + "content": [ + { + "type": "input_text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + tools_with_cache_control = [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object"}, + "cache_control": {"type": "ephemeral"}, + } + ] + input_before = copy.deepcopy(input_with_cache_control) + tools_before = copy.deepcopy(tools_with_cache_control) + + result = self.config.transform_responses_api_request( + model=self.model, + input=input_with_cache_control, + response_api_optional_request_params={"tools": tools_with_cache_control}, + litellm_params={}, + headers={}, + ) + + # the outbound body is still stripped + assert "cache_control" not in json.dumps(result) + # and the caller's objects are untouched, nested content blocks included + assert input_with_cache_control == input_before + assert tools_with_cache_control == tools_before + def test_transform_preserves_input_without_cache_control(self): """Inputs without cache_control must pass through unmodified.""" input_clean = [ @@ -2246,3 +2292,4 @@ class TestReasoningFollowsModelSupport: drop_params=True, ) assert mapped["reasoning"] == {"effort": "medium"} +import copy