Merge 2750687dc4 into dab2deb5ed

litellm/llms/openai/chat/gpt_transformation.py

@@ -57,7 +57,7 @@ from litellm.types.utils import (
 )
 from litellm.utils import convert_to_model_response_object
 
-from ..common_utils import OpenAIError
+from ..common_utils import OpenAIError, without_cache_control
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -387,23 +387,19 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         messages: list[AllMessageValues],
         tools: list["ChatCompletionToolParam"] | None = None,
     ) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
-        from litellm.litellm_core_utils.prompt_templates.common_utils import (
-            filter_value_from_dict,
-        )
         from litellm.types.llms.openai import ChatCompletionToolParam
 
-        for i, message in enumerate(messages):
-            messages[i] = cast(
-                AllMessageValues,
-                filter_value_from_dict(message, "cache_control"),
-            )
-        if tools is not None:
-            for i, tool in enumerate(tools):
-                tools[i] = cast(
-                    ChatCompletionToolParam,
-                    filter_value_from_dict(tool, "cache_control"),
-                )
-        return messages, tools
+        new_messages: Final = [  # mutable-ok: the declared return type is list[AllMessageValues]
+            cast(AllMessageValues, without_cache_control(message)) for message in messages
+        ]
+        new_tools: Final = (
+            [  # mutable-ok: the declared return type is list[ChatCompletionToolParam]
+                cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools
+            ]
+            if tools is not None
+            else None
+        )
+        return new_messages, new_tools
 
     def _targets_openai_hosted_endpoint(
         self,

litellm/llms/openai/common_utils.py

@@ -2,6 +2,7 @@
 Common helpers / utils across al OpenAI endpoints
 """
 
+import copy
 import hashlib
 import inspect
 import json
@@ -36,6 +37,37 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 
 
+def _contains_cache_control(value: object) -> bool:
+    # Iterative rather than recursive: the code-quality gate bans unignored recursion here,
+    # and request payloads are attacker-shaped, so an explicit worklist has no stack depth to blow.
+    pending: list[object] = [value]  # mutable-ok: a local worklist, never returned or stored
+    while pending:
+        node = pending.pop()  # rebind-ok: the loop variable of an explicit worklist
+        if isinstance(node, dict):
+            if "cache_control" in node:
+                return True
+            pending.extend(node.values())
+        elif isinstance(node, list):
+            pending.extend(node)
+    return False
+
+
+def without_cache_control(item: object) -> object:
+    """Return `item` with every `cache_control` key removed, without touching the original.
+
+    `filter_value_from_dict` deletes the key in place and recurses into nested dicts and
+    lists, so the caller's own message, tool or input object has to be copied first: it
+    may reuse the same list on a provider that does support prompt caching. The copy is
+    skipped when there is nothing to strip, which keeps the object identity the
+    pass-through paths rely on and avoids a deep copy on every request.
+    """
+    from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
+
+    if not isinstance(item, dict) or not _contains_cache_control(item):
+        return item
+    return filter_value_from_dict(copy.deepcopy(item), "cache_control")
+
+
 def _get_client_init_params(cls: type) -> tuple[str, ...]:
     """Extract __init__ parameter names (excluding 'self') from a class."""
     return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")

litellm/llms/openai/responses/transformation.py

@@ -30,7 +30,7 @@ from litellm.types.responses.main import *
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
-from ..common_utils import OpenAIError
+from ..common_utils import OpenAIError, without_cache_control
 from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
 
 OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
@@ -371,24 +371,30 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         the chat path. Strips Anthropic-only `cache_control` markers from
         Responses API input content blocks and tools.
 
-        `filter_value_from_dict` mutates each dict in place, so the same
-        objects are returned.
+        `filter_value_from_dict` deletes the key in place and recurses, so each
+        item is copied first: the caller keeps its own input list and may reuse
+        it on a provider that does support prompt caching.
         """
-        from litellm.litellm_core_utils.prompt_templates.common_utils import (
-            filter_value_from_dict,
+        new_input: Final = (
+            cast(  # cast-ok: the comprehension rebuilds the input item for item
+                ResponseInputParam,
+                [  # mutable-ok: the declared return type is ResponseInputParam
+                    without_cache_control(item) for item in input
+                ],
+            )
+            if isinstance(input, list)
+            else input
         )
 
-        if isinstance(input, list):
-            for item in input:
-                if isinstance(item, dict):
-                    filter_value_from_dict(cast(dict, item), "cache_control")
+        new_tools: Final = (
+            [  # mutable-ok: the declared return type is List[ALL_RESPONSES_API_TOOL_PARAMS]
+                without_cache_control(tool) for tool in tools
+            ]
+            if tools is not None
+            else None
+        )
 
-        if tools is not None:
-            for tool in tools:
-                if isinstance(tool, dict):
-                    filter_value_from_dict(cast(dict, tool), "cache_control")
-
-        return input, tools
+        return new_input, new_tools
 
     def _drop_foreign_tool_call_item_ids(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
         if self.custom_llm_provider not in _PROVIDERS_VALIDATING_TOOL_CALL_ITEM_IDS or not isinstance(input, list):

tests/unit/llms/openai/chat/test_openai_gpt_transformation.py

@@ -2,6 +2,9 @@
 Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py)
 """
 
+import copy
+import json
+
 
 import pytest
 from typing import Final
@@ -1406,3 +1409,107 @@ class TestToolSchemaCombinatorFlatteningForOpenAI:
         )
 
         assert request["tools"][0] is tool
+
+
+class TestCacheControlStrippingDoesNotMutateCallerInput:
+    """
+    Stripping cache_control for a provider that cannot use it must not reach back
+    into the caller's own message and tool objects.
+
+    filter_value_from_dict deletes the key in place and recurses into nested dicts
+    and lists, and remove_cache_control_flag_from_messages_and_tools assigned the
+    result back into the caller's list, so one call to any OpenAI-compatible
+    provider permanently stripped cache_control from a list the caller still holds.
+    Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt
+    caching, with no error and full-price billing.
+    """
+
+    def setup_method(self):
+        self.config = OpenAIGPTConfig()
+
+    @pytest.fixture(autouse=True)
+    def _clean_openai_base_env(self, monkeypatch):
+        monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
+        monkeypatch.delenv("OPENAI_API_BASE", raising=False)
+        monkeypatch.setattr(litellm, "api_base", None, raising=False)
+
+    @staticmethod
+    def _messages():
+        return [
+            {
+                "role": "system",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "a long cached system prompt",
+                        "cache_control": {"type": "ephemeral"},
+                    }
+                ],
+            },
+            {
+                "role": "user",
+                "content": "Hello",
+                "cache_control": {"type": "ephemeral"},
+            },
+        ]
+
+    @staticmethod
+    def _tools():
+        return [
+            {
+                "type": "function",
+                "function": {"name": "get_weather", "parameters": {"type": "object"}},
+                "cache_control": {"type": "ephemeral"},
+            }
+        ]
+
+    def _transform(self, messages, tools):
+        return self.config.transform_request(
+            model="gpt-4o",
+            messages=messages,
+            optional_params={"tools": tools},
+            litellm_params={"custom_llm_provider": "openai", "api_base": None},
+            headers={},
+        )
+
+    def test_caller_messages_and_tools_keep_cache_control(self):
+        messages, tools = self._messages(), self._tools()
+        messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools)
+
+        request = self._transform(messages, tools)
+
+        # the outbound body must still be stripped, both message-level and nested
+        assert "cache_control" not in json.dumps(request)
+        # and the caller's objects must be untouched, nested content blocks included
+        assert messages == messages_before
+        assert tools == tools_before
+
+    def test_a_later_anthropic_call_still_sees_cache_control(self):
+        """The user-visible consequence: the same message list reused on a provider
+        that does support caching must still carry the cache breakpoints."""
+        messages = self._messages()
+
+        self._transform(messages, self._tools())
+
+        anthropic_body = litellm.AnthropicConfig().transform_request(
+            model="claude-3-5-sonnet-20240620",
+            messages=messages,
+            optional_params={},
+            litellm_params={},
+            headers={},
+        )
+        assert "cache_control" in json.dumps(anthropic_body)
+
+    @staticmethod
+    def _artifact_tool():
+        return {
+            "type": "function",
+            "function": {
+                "name": "Artifact",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"field": {"type": "string", "pattern": _ARTIFACT_FIELD_PATTERN}},
+                    "required": ["field"],
+                },
+            },
+        }

tests/unit/llms/openai/responses/test_openai_responses_transformation.py

@@ -202,6 +202,52 @@ class TestOpenAIResponsesAPIConfig:
         assert "cache_control" not in result["tools"][0]
         assert result["tools"][0]["name"] == "get_weather"
 
+    def test_transform_does_not_strip_cache_control_from_the_callers_input(self):
+        """Stripping for OpenAI must not reach back into the caller's own objects.
+
+        `filter_value_from_dict` deletes the key in place and recurses, and
+        `_validate_input_param` passes plain dict items through by reference, so
+        the caller's input list used to lose its cache breakpoints. Reusing that
+        list on Anthropic or Bedrock afterwards then silently lost prompt caching.
+        Same defect as the Chat Completions path.
+        """
+        input_with_cache_control = [
+            {
+                "role": "system",
+                "content": [
+                    {
+                        "type": "input_text",
+                        "text": "a long cached system prompt",
+                        "cache_control": {"type": "ephemeral"},
+                    }
+                ],
+            }
+        ]
+        tools_with_cache_control = [
+            {
+                "type": "function",
+                "name": "get_weather",
+                "parameters": {"type": "object"},
+                "cache_control": {"type": "ephemeral"},
+            }
+        ]
+        input_before = copy.deepcopy(input_with_cache_control)
+        tools_before = copy.deepcopy(tools_with_cache_control)
+
+        result = self.config.transform_responses_api_request(
+            model=self.model,
+            input=input_with_cache_control,
+            response_api_optional_request_params={"tools": tools_with_cache_control},
+            litellm_params={},
+            headers={},
+        )
+
+        # the outbound body is still stripped
+        assert "cache_control" not in json.dumps(result)
+        # and the caller's objects are untouched, nested content blocks included
+        assert input_with_cache_control == input_before
+        assert tools_with_cache_control == tools_before
+
     def test_transform_preserves_input_without_cache_control(self):
         """Inputs without cache_control must pass through unmodified."""
         input_clean = [
@@ -2246,3 +2292,4 @@ class TestReasoningFollowsModelSupport:
             drop_params=True,
         )
         assert mapped["reasoning"] == {"effort": "medium"}
+import copy