From 5215469e3331debb694a3220c59b0bb0cd7c5e41 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Date: Thu, 27 Aug 2026 10:09:19 -0700 Subject: [PATCH] fix(openai): stop stripping cache_control from the caller's own messages and tools filter_value_from_dict deletes the key in place and recurses, so stripping cache_control for a provider that cannot use it reached back into the caller's own message and tool objects. One OpenAI-compatible call then killed Anthropic and Bedrock prompt caching on every later call that reused the same list. Copy each item before filtering, on both the chat and responses paths, and build each replacement in a single Final assignment so the type-discipline budget is unchanged. Rebased onto the current staging tip, which had removed the `# type: ignore` comments and the tests' sys.path bootstrap in the meantime. Signed-off-by: Vineeth Sai --- .../llms/openai/chat/gpt_transformation.py | 24 +++-- .../llms/openai/responses/transformation.py | 39 +++++--- .../chat/test_openai_gpt_transformation.py | 92 +++++++++++++++++++ .../test_openai_responses_transformation.py | 47 ++++++++++ 4 files changed, 183 insertions(+), 19 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 16fd042cb2f..5465ef539b2 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -2,6 +2,7 @@ Support for gpt model family """ +import copy import json import os from collections.abc import AsyncIterator, Coroutine, Iterator @@ -376,18 +377,25 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) from litellm.types.llms.openai import ChatCompletionToolParam - for i, message in enumerate(messages): - messages[i] = cast( + new_messages: Final = [ # mutable-ok: the declared return type is list[AllMessageValues] + cast( AllMessageValues, - filter_value_from_dict(message, "cache_control"), + filter_value_from_dict(copy.deepcopy(message), "cache_control"), ) - if tools is not None: - for i, tool in enumerate(tools): - tools[i] = cast( + for message in messages + ] + new_tools: Final = ( + [ # mutable-ok: the declared return type is list[ChatCompletionToolParam] + cast( ChatCompletionToolParam, - filter_value_from_dict(tool, "cache_control"), + filter_value_from_dict(copy.deepcopy(tool), "cache_control"), ) - return messages, tools + for tool in tools + ] + if tools is not None + else None + ) + return new_messages, new_tools def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index b2a69564908..c786ab54a6b 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -1,3 +1,4 @@ +import copy from typing import TYPE_CHECKING, Any, Final, cast, get_type_hints import httpx @@ -174,24 +175,40 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): the chat path. Strips Anthropic-only `cache_control` markers from Responses API input content blocks and tools. - `filter_value_from_dict` mutates each dict in place, so the same - objects are returned. + `filter_value_from_dict` deletes the key in place and recurses, so each + item is copied first: the caller keeps its own input list and may reuse + it on a provider that does support prompt caching. """ from litellm.litellm_core_utils.prompt_templates.common_utils import ( filter_value_from_dict, ) - if isinstance(input, list): - for item in input: - if isinstance(item, dict): - filter_value_from_dict(cast(dict, item), "cache_control") + new_input: Final = ( + cast( # cast-ok: the comprehension rebuilds the input item for item + ResponseInputParam, + [ # mutable-ok: the declared return type is ResponseInputParam + filter_value_from_dict(copy.deepcopy(cast(dict, item)), "cache_control") + if isinstance(item, dict) + else item + for item in input + ], + ) + if isinstance(input, list) + else input + ) - if tools is not None: - for tool in tools: - if isinstance(tool, dict): - filter_value_from_dict(cast(dict, tool), "cache_control") + new_tools: Final = ( + [ # mutable-ok: the declared return type is List[ALL_RESPONSES_API_TOOL_PARAMS] + filter_value_from_dict(copy.deepcopy(cast(dict, tool)), "cache_control") + if isinstance(tool, dict) + else tool + for tool in tools + ] + if tools is not None + else None + ) - return input, tools + return new_input, new_tools def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam: """ diff --git a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py index f4c38f8f797..88d59dda60d 100644 --- a/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/test_litellm/llms/openai/chat/test_openai_gpt_transformation.py @@ -2,6 +2,8 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ +import copy +import json import pytest @@ -912,3 +914,93 @@ class TestOpenAIPromptCacheBreakpointChatPath: assert request["messages"][1]["content"] == [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}] assert request["extra_body"] == {"prompt_cache_options": self.EXPLICIT} assert "prompt_cache_options" not in request + + +class TestCacheControlStrippingDoesNotMutateCallerInput: + """ + Stripping cache_control for a provider that cannot use it must not reach back + into the caller's own message and tool objects. + + filter_value_from_dict deletes the key in place and recurses into nested dicts + and lists, and remove_cache_control_flag_from_messages_and_tools assigned the + result back into the caller's list, so one call to any OpenAI-compatible + provider permanently stripped cache_control from a list the caller still holds. + Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt + caching, with no error and full-price billing. + """ + + def setup_method(self): + self.config = OpenAIGPTConfig() + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _messages(): + return [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": "Hello", + "cache_control": {"type": "ephemeral"}, + }, + ] + + @staticmethod + def _tools(): + return [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + def _transform(self, messages, tools): + return self.config.transform_request( + model="gpt-4o", + messages=messages, + optional_params={"tools": tools}, + litellm_params={"custom_llm_provider": "openai", "api_base": None}, + headers={}, + ) + + def test_caller_messages_and_tools_keep_cache_control(self): + messages, tools = self._messages(), self._tools() + messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools) + + request = self._transform(messages, tools) + + # the outbound body must still be stripped, both message-level and nested + assert "cache_control" not in json.dumps(request) + # and the caller's objects must be untouched, nested content blocks included + assert messages == messages_before + assert tools == tools_before + + def test_a_later_anthropic_call_still_sees_cache_control(self): + """The user-visible consequence: the same message list reused on a provider + that does support caching must still carry the cache breakpoints.""" + messages = self._messages() + + self._transform(messages, self._tools()) + + anthropic_body = litellm.AnthropicConfig().transform_request( + model="claude-3-5-sonnet-20240620", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "cache_control" in json.dumps(anthropic_body) diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py index c03c632363d..7b8cbc5a087 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_transformation.py @@ -1,3 +1,4 @@ +import copy import json from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -200,6 +201,52 @@ class TestOpenAIResponsesAPIConfig: assert "cache_control" not in result["tools"][0] assert result["tools"][0]["name"] == "get_weather" + def test_transform_does_not_strip_cache_control_from_the_callers_input(self): + """Stripping for OpenAI must not reach back into the caller's own objects. + + `filter_value_from_dict` deletes the key in place and recurses, and + `_validate_input_param` passes plain dict items through by reference, so + the caller's input list used to lose its cache breakpoints. Reusing that + list on Anthropic or Bedrock afterwards then silently lost prompt caching. + Same defect as the Chat Completions path. + """ + input_with_cache_control = [ + { + "role": "system", + "content": [ + { + "type": "input_text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + tools_with_cache_control = [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object"}, + "cache_control": {"type": "ephemeral"}, + } + ] + input_before = copy.deepcopy(input_with_cache_control) + tools_before = copy.deepcopy(tools_with_cache_control) + + result = self.config.transform_responses_api_request( + model=self.model, + input=input_with_cache_control, + response_api_optional_request_params={"tools": tools_with_cache_control}, + litellm_params={}, + headers={}, + ) + + # the outbound body is still stripped + assert "cache_control" not in json.dumps(result) + # and the caller's objects are untouched, nested content blocks included + assert input_with_cache_control == input_before + assert tools_with_cache_control == tools_before + def test_transform_preserves_input_without_cache_control(self): """Inputs without cache_control must pass through unmodified.""" input_clean = [