From bcaa37f9920b7f08c3769e45cd186bb567ccfdd2 Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Wed, 23 Sep 2026 11:13:54 +0700 Subject: [PATCH 01/10] fix(anthropic): write the cache_control marker on a block that accepts one The marker went on the last content block whatever it was. Anthropic accepts one on text, image, tool_use, tool_result and document blocks; a thinking block is not among them, and an empty text block is replaced by a placeholder before the request goes out, taking the marker with it. Either way the configured breakpoint is spent and the turn is not cached. Walk back to the last block that accepts a marker, the way the OpenAI prompt cache path in this file already does. A tool message is the exception: its empty text block is kept, nested in the tool_result the conversion builds, so the marker stays on it. --- .../anthropic_cache_control_hook.py | 36 +++- .../test_anthropic_cache_control_hook.py | 160 ++++++++++++++++++ 2 files changed, 192 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4e473c2f3f0..72972f2f25c 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -84,6 +84,10 @@ _O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) +# Anthropic lists the block types a cache_control marker may sit on: text, image, +# tool_use, tool_result and document. A list of refused types rather than accepted +# ones so a block type this code has not been told about still takes a marker. +ANTHROPIC_BLOCK_TYPES_WITHOUT_CACHE_CONTROL: Final = frozenset({"thinking", "redacted_thinking"}) OPENAI_API_HOST: Final = "api.openai.com" OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE") _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) @@ -767,6 +771,28 @@ def _message_dropping_created_cache_controls_between( ) +def _index_of_block_accepting_cache_control(content: list[object], on_a_tool_message: bool) -> int | None: + """Position of the last block a cache_control marker can be written on, or None. + + Searched from the end: a marker caches everything up to and including its own + block, so the last one caches the most. + + An empty text block is refused because the rewrites this hook runs ahead of drop + it, and the marker goes with it. ``on_a_tool_message`` lifts that refusal: a tool + message keeps its empty block, nested in the tool_result the rewrite builds. + """ + for index in range(len(content) - 1, -1, -1): + block = content[index] + if not isinstance(block, dict): + continue + if block.get("type") in ANTHROPIC_BLOCK_TYPES_WITHOUT_CACHE_CONTROL: + continue + if block.get("type") == "text" and not block.get("text") and not on_a_tool_message: + continue + return index + return None + + # Set by a caller whose message list is not the one that goes upstream -- today the # Responses API layer, whose `instructions` only becomes a system message further down. # Tells this hook to hand role-targeted points to the pass holding the final messages @@ -1228,11 +1254,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): # 1. if string, insert cache control in the message if isinstance(message_content, str): message["cache_control"] = control - # 2. list of objects - only apply to the last item Claude accepts a breakpoint on + # 2. list of objects - the last block that accepts a marker, per Anthropic spec elif isinstance(message_content, list): - markable_index: Final = _last_index_accepting_cache_control(message_content) - if markable_index is not None: - message_content[markable_index]["cache_control"] = control # pyright: ignore[reportGeneralTypeIssues] # loose runtime dict + target_index: Final = _index_of_block_accepting_cache_control( + message_content, on_a_tool_message=message.get("role") == "tool" + ) + if target_index is not None: + message_content[target_index]["cache_control"] = control # pyright: ignore[reportGeneralTypeIssues] # loose runtime dict return message @staticmethod diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 766038d5c4f..3be6d5c83d9 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -5863,3 +5863,163 @@ def test_cache_control_hook_reads_a_non_string_answer_id_as_answering_no_tool_ca assert _cache_marks_after_injection(_search_conversation_with(malformed)) == _cache_marks_after_injection( _search_conversation_with(well_formed) ) + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_skips_a_thinking_block(monkeypatch: pytest.MonkeyPatch): + """ + A cache_control marker on a thinking block is spent on a block Anthropic does not + accept one on, so the turn it was meant to cache is not cached. The marker goes on + the last block of the message that accepts one instead. + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake_anthropic_key") + anthropic_cache_control_hook = AnthropicCacheControlHook() + monkeypatch.setattr(litellm, "callbacks", [anthropic_cache_control_hook]) + + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Because two plus two is four."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, + } + mock_response.status_code = 200 + + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=mock_response) as mock_post: + await litellm.acompletion( + model="anthropic/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "What is 2 + 2?"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "The answer is 4."}, + {"type": "thinking", "thinking": "Adding two and two.", "signature": "sig"}, + {"type": "redacted_thinking", "data": "redacted"}, + ], + }, + {"role": "user", "content": "Why?"}, + ], + cache_control_injection_points=[{"location": "message", "index": 1}], + client=client, + ) + + request_body = mock_post.call_args.kwargs["json"] + + assert request_body["messages"][1] == { + "role": "assistant", + "content": [ + {"type": "text", "text": "The answer is 4.", "cache_control": {"type": "ephemeral"}}, + {"type": "thinking", "thinking": "Adding two and two.", "signature": "sig"}, + ], + } + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_skips_an_empty_text_block(monkeypatch: pytest.MonkeyPatch): + """ + An empty text block is replaced by a placeholder before the request goes out, and a + cache_control marker written on it is replaced with it. The marker goes on the last + block that survives instead. + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake_anthropic_key") + monkeypatch.setattr(litellm, "callbacks", [AnthropicCacheControlHook()]) + + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Because two plus two is four."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, + } + mock_response.status_code = 200 + + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=mock_response) as mock_post: + await litellm.acompletion( + model="anthropic/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "What is 2 + 2?"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "The answer is 4."}, + {"type": "text", "text": ""}, + ], + }, + {"role": "user", "content": "Why?"}, + ], + cache_control_injection_points=[{"location": "message", "index": 1}], + client=client, + ) + + request_body = mock_post.call_args.kwargs["json"] + + assert request_body["messages"][1] == { + "role": "assistant", + "content": [ + {"type": "text", "text": "The answer is 4.", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "[System: Empty message content sanitised to satisfy protocol]"}, + ], + } + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_marks_an_empty_tool_result(monkeypatch: pytest.MonkeyPatch): + """ + A tool message keeps its empty text block - it goes out nested in the tool_result the + conversion builds - so the marker stays on it rather than walking off the message. + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake_anthropic_key") + monkeypatch.setattr(litellm, "callbacks", [AnthropicCacheControlHook()]) + + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Nothing came back."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, + } + mock_response.status_code = 200 + + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=mock_response) as mock_post: + await litellm.acompletion( + model="anthropic/claude-sonnet-4-5", + messages=[ + {"role": "user", "content": "Search for it."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "search", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": ""}]}, + ], + cache_control_injection_points=[{"location": "message", "index": 2}], + client=client, + ) + + request_body = mock_post.call_args.kwargs["json"] + + assert request_body["messages"][-1] == { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_1", + "content": [{"type": "text", "text": "", "cache_control": {"type": "ephemeral"}}], + } + ], + } From b7b481d89d05f24d03d4fba0c8fed2d84e0dec2a Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Wed, 23 Sep 2026 12:31:31 +0700 Subject: [PATCH 02/10] fix(anthropic): count an injection point's index within the role it names {"role": "assistant", "index": -1} read as "the last message", because the index branch returned before the role was looked at. On a conversation ending with a user turn it marked that turn; the assistant turn the point named went uncached. The message it lands on may also have nowhere to write a marker. An assistant turn that said everything through tool_calls has no content, and it is the newest assistant turn at every step of an agent loop, so the point was spent and nothing was written. Count the index within the role's turns, and walk back from there to the nearest earlier turn that accepts a marker. The walk stays inside the role the point named. --- .../anthropic_cache_control_hook.py | 63 ++++- .../test_anthropic_cache_control_hook.py | 259 ++++++++++++++++++ 2 files changed, 309 insertions(+), 13 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 72972f2f25c..6325f86f2f9 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -793,6 +793,25 @@ def _index_of_block_accepting_cache_control(content: list[object], on_a_tool_mes return None +def _message_accepts_cache_control(message: object) -> bool: + """Whether a cache_control marker written on this message reaches the provider. + + Reads the same block rule as `_safe_insert_cache_control_in_message`, so the two + cannot disagree about where a marker goes. A message whose content is empty -- + an assistant turn that said everything through ``tool_calls`` -- has nowhere to + put one. + """ + if not isinstance(message, dict): + return False + on_a_tool_message: Final = message.get("role") == "tool" + content: Final = message.get("content") + if isinstance(content, str): + return content != "" or on_a_tool_message + if isinstance(content, list): + return _index_of_block_accepting_cache_control(content, on_a_tool_message) is not None + return False + + # Set by a caller whose message list is not the one that goes upstream -- today the # Responses API layer, whose `instructions` only becomes a system message further down. # Tells this hook to hand role-targeted points to the pass holding the final messages @@ -1179,28 +1198,46 @@ class AnthropicCacheControlHook(CustomPromptManagement): else: targetted_index = _targetted_index - # Case 1: Target by specific index - if targetted_index is not None: - original_index: Final = targetted_index - if targetted_index < 0: - targetted_index += len(messages) + # The messages the point names. A point with a role counts its index within + # that role's turns, so {role: assistant, index: -1} is the last assistant + # turn rather than the last message. + targetted_role: Final = point.get("role", None) + candidates: Final = [ + index + for index, message in enumerate(messages) + if targetted_role is None or message.get("role") == targetted_role + ] - if 0 <= targetted_index < len(messages): - return [targetted_index] + # Case 1: Target by role alone + if targetted_index is None: + if targetted_role is None: + return [] + return [index for index in candidates if _message_accepts_cache_control(messages[index])] + # Case 2: Target by index, within the role the point named + original_index: Final = targetted_index + if targetted_index < 0: + targetted_index += len(candidates) + + if not 0 <= targetted_index < len(candidates): verbose_logger.warning( "AnthropicCacheControlHook: Provided index %s is out of bounds for message list of length %s. Targeted index was %s. Skipping cache control injection for this point.", original_index, - len(messages), + len(candidates), targetted_index, ) return [] - # Case 2: Target by role - targetted_role: Final = point.get("role", None) - if targetted_role is not None: - return [idx for idx, msg in enumerate(messages) if msg.get("role") == targetted_role] - + # The message it arrives at may have nowhere to write a marker -- an assistant + # turn that said everything through tool_calls is the common one -- so walk back + # to the nearest earlier turn that does. Stopping there would spend the point + # and write nothing. + position = targetted_index + while position >= 0: + index = candidates[position] + if _message_accepts_cache_control(messages[index]): + return [index] + position -= 1 return [] @staticmethod diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 3be6d5c83d9..e6b16b4a7b6 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -6023,3 +6023,262 @@ async def test_anthropic_cache_control_hook_marks_an_empty_tool_result(monkeypat } ], } + + +def _anthropic_response_mock() -> MagicMock: + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "Sure."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, + } + mock_response.status_code = 200 + return mock_response + + +async def _marked_messages(messages, points, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake_anthropic_key") + monkeypatch.setattr(litellm, "callbacks", [AnthropicCacheControlHook()]) + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=_anthropic_response_mock()) as mock_post: + await litellm.acompletion( + model="anthropic/claude-sonnet-4-5", + messages=messages, + cache_control_injection_points=points, + client=client, + ) + return mock_post.call_args.kwargs["json"]["messages"] + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_counts_the_index_within_the_role(monkeypatch: pytest.MonkeyPatch): + """ + {"role": "assistant", "index": -1} means the last assistant turn. Counting the index + over every message instead marks whatever message happens to be last, so the turn the + point named is not cached. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "What is 2 + 2?"}, + {"role": "assistant", "content": "The answer is 4."}, + {"role": "user", "content": "Why?"}, + ], + [{"location": "message", "role": "assistant", "index": -1}], + monkeypatch, + ) + + assert marked == [ + {"role": "user", "content": [{"type": "text", "text": "What is 2 + 2?"}]}, + { + "role": "assistant", + "content": [{"type": "text", "text": "The answer is 4.", "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": [{"type": "text", "text": "Why?"}]}, + ] + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_walks_back_off_a_tool_call_turn(monkeypatch: pytest.MonkeyPatch): + """ + An assistant turn that said everything through tool_calls has no content to mark, and + it is the newest assistant turn at every step of an agent loop. Stopping there spends + the point and writes nothing, so the walk goes back to the turn before it. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "Open the file."}, + {"role": "assistant", "content": "Opening it now."}, + {"role": "user", "content": "Thanks."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "open_file", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "file contents"}, + ], + [{"location": "message", "role": "assistant", "index": -1}], + monkeypatch, + ) + + assert marked[1] == { + "role": "assistant", + "content": [{"type": "text", "text": "Opening it now.", "cache_control": {"type": "ephemeral"}}], + } + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_walk_back_stays_in_the_role(monkeypatch: pytest.MonkeyPatch): + """ + The walk stays inside the role the point named, so an empty last user turn marks an + earlier user turn rather than the assistant turn between them. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "first question"}, + {"role": "assistant", "content": "first answer"}, + {"role": "user", "content": ""}, + ], + [{"location": "message", "role": "user", "index": -1}], + monkeypatch, + ) + + assert marked[0] == { + "role": "user", + "content": [{"type": "text", "text": "first question", "cache_control": {"type": "ephemeral"}}], + } + assert marked[1] == {"role": "assistant", "content": [{"type": "text", "text": "first answer"}]} + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_bounds_the_index_by_the_role(monkeypatch: pytest.MonkeyPatch): + """ + An index is in bounds when the role has that many turns, not when the message list + does. -3 against two assistant turns names no turn, so the point marks nothing. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + {"role": "assistant", "content": "a2"}, + ], + [{"location": "message", "role": "assistant", "index": -3}], + monkeypatch, + ) + + assert marked == [ + {"role": "user", "content": [{"type": "text", "text": "u1"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, + {"role": "user", "content": [{"type": "text", "text": "u2"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "a2"}]}, + ] + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_bounds_a_positive_index_by_the_role(monkeypatch: pytest.MonkeyPatch): + """ + Index 3 names a fourth turn of the role. Two assistant turns among five messages is + out of bounds, and bounding it by the message list instead reads past the end of the + role's turns. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "u2"}, + {"role": "assistant", "content": "a2"}, + {"role": "user", "content": "u3"}, + ], + [{"location": "message", "role": "assistant", "index": 3}], + monkeypatch, + ) + + assert marked == [ + {"role": "user", "content": [{"type": "text", "text": "u1"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "a1"}]}, + {"role": "user", "content": [{"type": "text", "text": "u2"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "a2"}]}, + {"role": "user", "content": [{"type": "text", "text": "u3"}]}, + ] + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_role_alone_skips_a_turn_with_nowhere_to_write( + monkeypatch: pytest.MonkeyPatch, +): + """ + A point naming a role and no index marks every turn of that role that accepts a + marker. An empty turn would spend a breakpoint on a block the conversion replaces. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "u1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": ""}, + {"role": "assistant", "content": "a2"}, + {"role": "user", "content": "u3"}, + ], + [{"location": "message", "role": "user"}], + monkeypatch, + ) + + assert marked[0] == { + "role": "user", + "content": [{"type": "text", "text": "u1", "cache_control": {"type": "ephemeral"}}], + } + assert marked[2] == { + "role": "user", + "content": [{"type": "text", "text": "[System: Empty message content sanitised to satisfy protocol]"}], + } + assert marked[4] == { + "role": "user", + "content": [{"type": "text", "text": "u3", "cache_control": {"type": "ephemeral"}}], + } + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_marks_a_tool_turn_that_returned_nothing( + monkeypatch: pytest.MonkeyPatch, +): + """ + A tool that returned nothing is the newest turn of an agent loop. Its empty content + reaches the provider inside the tool_result, so the marker goes on it rather than + walking back and leaving the tool_use outside the cached prefix. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "Search for it."}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "search", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": ""}, + ], + [{"location": "message", "index": 2}], + monkeypatch, + ) + + assert marked[-1] == { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_1", + "content": "", + "cache_control": {"type": "ephemeral"}, + } + ], + } + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_walks_back_off_a_thinking_only_turn(monkeypatch: pytest.MonkeyPatch): + """ + A turn whose only block is a thinking block has no block that accepts a marker, so + the walk goes back to the turn before it rather than spending the point there. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "q1"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "q2"}, + {"role": "assistant", "content": [{"type": "thinking", "thinking": "t", "signature": "s"}]}, + ], + [{"location": "message", "role": "assistant", "index": -1}], + monkeypatch, + ) + + assert marked[1] == { + "role": "assistant", + "content": [{"type": "text", "text": "a1", "cache_control": {"type": "ephemeral"}}], + } + assert marked[3] == {"role": "assistant", "content": [{"type": "thinking", "thinking": "t", "signature": "s"}]} From 3c0b9eef8e605184ab0fc0a410751f9c808fc97f Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Wed, 23 Sep 2026 12:40:20 +0700 Subject: [PATCH 03/10] fix(anthropic): keep two injection points as two cache breakpoints The second point resolved to the message the first one had just marked, found it marked and was dropped. Two configured breakpoints became one, and the four exist so that a prefix which stops matching at one can still match at an earlier one. Carry the messages already marked in this request into the walk, so a point that arrives on one keeps going back instead of being spent there. --- .../anthropic_cache_control_hook.py | 19 ++++++++++--- .../test_anthropic_cache_control_hook.py | 27 +++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 6325f86f2f9..f11cb752309 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -1147,6 +1147,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): messages, answered_tool_ids=client_answers ) + taken: set[int] = set() limit_reached = False for point in points: if used_blocks >= max_blocks: @@ -1157,7 +1158,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): type="ephemeral" ) - for target_index in AnthropicCacheControlHook._resolve_target_indices(point=point, messages=messages): + for target_index in AnthropicCacheControlHook._resolve_target_indices( + point=point, messages=messages, taken=taken + ): if used_blocks >= max_blocks: limit_reached = True break @@ -1171,6 +1174,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) if AnthropicCacheControlHook._message_has_cache_control(messages[target_index], client_answers): used_blocks += 1 + taken.add(target_index) if limit_reached: break @@ -1185,9 +1189,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def _resolve_target_indices( - point: CacheControlMessageInjectionPoint, messages: list[AllMessageValues] + point: CacheControlMessageInjectionPoint, messages: list[AllMessageValues], taken: set[int] | None = None ) -> list[int]: - """Resolve which message indices an injection point targets.""" + """Resolve which message indices an injection point targets. + + ``taken`` is the messages an earlier point in the same request already marked. + The walk goes past them: arriving on one, finding it marked and dropping the + point turns two configured breakpoints into one, and the four exist so that a + prefix which stops matching at one can still match at an earlier one. + """ + already_marked: Final = taken or set() _targetted_index: Final[int | str | None] = point.get("index", None) targetted_index: int | None = None if isinstance(_targetted_index, str): @@ -1235,7 +1246,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): position = targetted_index while position >= 0: index = candidates[position] - if _message_accepts_cache_control(messages[index]): + if index not in already_marked and _message_accepts_cache_control(messages[index]): return [index] position -= 1 return [] diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index e6b16b4a7b6..bc90e50b84a 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -6282,3 +6282,30 @@ async def test_anthropic_cache_control_hook_walks_back_off_a_thinking_only_turn( "content": [{"type": "text", "text": "a1", "cache_control": {"type": "ephemeral"}}], } assert marked[3] == {"role": "assistant", "content": [{"type": "thinking", "thinking": "t", "signature": "s"}]} + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_two_points_stay_two_breakpoints(monkeypatch: pytest.MonkeyPatch): + """ + The second point walks past the message the first one marked. Arriving on it, + finding it marked and dropping the point turns two configured breakpoints into one, + and the four exist so a prefix that stops matching at one can match at an earlier one. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "open"}, + {"role": "assistant", "content": "ack"}, + {"role": "user", "content": ""}, + ], + [{"location": "message", "index": -1}, {"location": "message", "index": -2}], + monkeypatch, + ) + + assert marked == [ + {"role": "user", "content": [{"type": "text", "text": "open", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "ack", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "user", + "content": [{"type": "text", "text": "[System: Empty message content sanitised to satisfy protocol]"}], + }, + ] From d7e81274fe9c156d002595a4185689e933a7bd0a Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Wed, 23 Sep 2026 13:39:11 +0700 Subject: [PATCH 04/10] fix(anthropic): keep the type-discipline budget flat Build the candidate indices as a tuple, take the already-marked messages as a Container with a frozenset default, and say why the one remaining set is mutable. --- litellm/integrations/anthropic_cache_control_hook.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index f11cb752309..e29136b8244 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -1147,7 +1147,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): messages, answered_tool_ids=client_answers ) - taken: set[int] = set() + taken: set[int] = set() # mutable-ok: the messages this request has already marked limit_reached = False for point in points: if used_blocks >= max_blocks: @@ -1189,7 +1189,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def _resolve_target_indices( - point: CacheControlMessageInjectionPoint, messages: list[AllMessageValues], taken: set[int] | None = None + point: CacheControlMessageInjectionPoint, messages: list[AllMessageValues], taken: Container[int] = frozenset() ) -> list[int]: """Resolve which message indices an injection point targets. @@ -1198,7 +1198,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): point turns two configured breakpoints into one, and the four exist so that a prefix which stops matching at one can still match at an earlier one. """ - already_marked: Final = taken or set() _targetted_index: Final[int | str | None] = point.get("index", None) targetted_index: int | None = None if isinstance(_targetted_index, str): @@ -1213,11 +1212,11 @@ class AnthropicCacheControlHook(CustomPromptManagement): # that role's turns, so {role: assistant, index: -1} is the last assistant # turn rather than the last message. targetted_role: Final = point.get("role", None) - candidates: Final = [ + candidates: Final = tuple( index for index, message in enumerate(messages) if targetted_role is None or message.get("role") == targetted_role - ] + ) # Case 1: Target by role alone if targetted_index is None: @@ -1246,7 +1245,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): position = targetted_index while position >= 0: index = candidates[position] - if index not in already_marked and _message_accepts_cache_control(messages[index]): + if index not in taken and _message_accepts_cache_control(messages[index]): return [index] position -= 1 return [] From cb8dc9b9ffa1235fb9e15b103c912fd3c028185d Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Wed, 23 Sep 2026 14:19:15 +0700 Subject: [PATCH 05/10] fix(anthropic): refuse a marker on a tool_reference block, and drop the mutable state The tool_result conversion rebuilds a tool_reference from its type and tool_name alone, so a marker written on one is dropped before the request goes out. Refuse that block type the way thinking blocks are refused. Resolve an injection point against the messages that carry no breakpoint yet instead of carrying a mutable set of the ones this request marked, so the walk needs no reassignment and the caller's own markers are walked past too. --- .../anthropic_cache_control_hook.py | 81 +++++++------------ .../test_anthropic_cache_control_hook.py | 51 +++++++++++- 2 files changed, 77 insertions(+), 55 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index e29136b8244..70cd0887442 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -84,10 +84,12 @@ _O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) -# Anthropic lists the block types a cache_control marker may sit on: text, image, -# tool_use, tool_result and document. A list of refused types rather than accepted -# ones so a block type this code has not been told about still takes a marker. -ANTHROPIC_BLOCK_TYPES_WITHOUT_CACHE_CONTROL: Final = frozenset({"thinking", "redacted_thinking"}) +# Block types a marker never reaches the provider on. Anthropic accepts one on text, +# image, tool_use, tool_result and document, so a thinking block is refused there; a +# tool_reference is rebuilt from its type and tool_name alone by the tool_result +# conversion, which drops everything else. A list of refused types rather than +# accepted ones, so a block type this code has not been told about still takes one. +ANTHROPIC_BLOCK_TYPES_WITHOUT_CACHE_CONTROL: Final = frozenset({"thinking", "redacted_thinking", "tool_reference"}) OPENAI_API_HOST: Final = "api.openai.com" OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE") _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) @@ -772,14 +774,10 @@ def _message_dropping_created_cache_controls_between( def _index_of_block_accepting_cache_control(content: list[object], on_a_tool_message: bool) -> int | None: - """Position of the last block a cache_control marker can be written on, or None. + """Last block a marker written here reaches the provider on, searched from the end. - Searched from the end: a marker caches everything up to and including its own - block, so the last one caches the most. - - An empty text block is refused because the rewrites this hook runs ahead of drop - it, and the marker goes with it. ``on_a_tool_message`` lifts that refusal: a tool - message keeps its empty block, nested in the tool_result the rewrite builds. + An empty text block is replaced by a placeholder unless it sits on a tool message, + where it goes out inside the tool_result. """ for index in range(len(content) - 1, -1, -1): block = content[index] @@ -794,13 +792,7 @@ def _index_of_block_accepting_cache_control(content: list[object], on_a_tool_mes def _message_accepts_cache_control(message: object) -> bool: - """Whether a cache_control marker written on this message reaches the provider. - - Reads the same block rule as `_safe_insert_cache_control_in_message`, so the two - cannot disagree about where a marker goes. A message whose content is empty -- - an assistant turn that said everything through ``tool_calls`` -- has nowhere to - put one. - """ + """Whether a marker written on this message reaches the provider.""" if not isinstance(message, dict): return False on_a_tool_message: Final = message.get("role") == "tool" @@ -1147,7 +1139,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): messages, answered_tool_ids=client_answers ) - taken: set[int] = set() # mutable-ok: the messages this request has already marked limit_reached = False for point in points: if used_blocks >= max_blocks: @@ -1159,7 +1150,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) for target_index in AnthropicCacheControlHook._resolve_target_indices( - point=point, messages=messages, taken=taken + point=point, messages=messages, answered_tool_ids=client_answers ): if used_blocks >= max_blocks: limit_reached = True @@ -1174,7 +1165,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) if AnthropicCacheControlHook._message_has_cache_control(messages[target_index], client_answers): used_blocks += 1 - taken.add(target_index) if limit_reached: break @@ -1189,15 +1179,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def _resolve_target_indices( - point: CacheControlMessageInjectionPoint, messages: list[AllMessageValues], taken: Container[int] = frozenset() + point: CacheControlMessageInjectionPoint, messages: list[AllMessageValues], answered_tool_ids: Container[str] ) -> list[int]: - """Resolve which message indices an injection point targets. - - ``taken`` is the messages an earlier point in the same request already marked. - The walk goes past them: arriving on one, finding it marked and dropping the - point turns two configured breakpoints into one, and the four exist so that a - prefix which stops matching at one can still match at an earlier one. - """ + """Resolve which message indices an injection point targets.""" _targetted_index: Final[int | str | None] = point.get("index", None) targetted_index: int | None = None if isinstance(_targetted_index, str): @@ -1208,47 +1192,36 @@ class AnthropicCacheControlHook(CustomPromptManagement): else: targetted_index = _targetted_index - # The messages the point names. A point with a role counts its index within - # that role's turns, so {role: assistant, index: -1} is the last assistant - # turn rather than the last message. targetted_role: Final = point.get("role", None) candidates: Final = tuple( index for index, message in enumerate(messages) if targetted_role is None or message.get("role") == targetted_role ) + free: Final = frozenset( + index + for index in candidates + if not AnthropicCacheControlHook._message_has_cache_control(messages[index], answered_tool_ids) + and _message_accepts_cache_control(messages[index]) + ) # Case 1: Target by role alone if targetted_index is None: - if targetted_role is None: - return [] - return [index for index in candidates if _message_accepts_cache_control(messages[index])] + return [] if targetted_role is None else [index for index in candidates if index in free] - # Case 2: Target by index, within the role the point named - original_index: Final = targetted_index - if targetted_index < 0: - targetted_index += len(candidates) - - if not 0 <= targetted_index < len(candidates): + # Case 2: Target by index, counted within the role the point named + position: Final = targetted_index + len(candidates) if targetted_index < 0 else targetted_index + if not 0 <= position < len(candidates): verbose_logger.warning( "AnthropicCacheControlHook: Provided index %s is out of bounds for message list of length %s. Targeted index was %s. Skipping cache control injection for this point.", - original_index, - len(candidates), targetted_index, + len(candidates), + position, ) return [] - # The message it arrives at may have nowhere to write a marker -- an assistant - # turn that said everything through tool_calls is the common one -- so walk back - # to the nearest earlier turn that does. Stopping there would spend the point - # and write nothing. - position = targetted_index - while position >= 0: - index = candidates[position] - if index not in taken and _message_accepts_cache_control(messages[index]): - return [index] - position -= 1 - return [] + landing: Final = next((candidates[step] for step in range(position, -1, -1) if candidates[step] in free), None) + return [] if landing is None else [landing] @staticmethod def _count_cache_control_blocks(message: object, answered_tool_ids: Container[str] = frozenset()) -> int: diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index bc90e50b84a..8ec4431eaf0 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.types.integrations.anthropic_cache_control_hook import CacheControlMessageInjectionPoint from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantToolCall from litellm.types.utils import ChatCompletionMessageToolCall, LlmProviders, Message from litellm.utils import ProviderConfigManager @@ -6040,7 +6041,11 @@ def _anthropic_response_mock() -> MagicMock: return mock_response -async def _marked_messages(messages, points, monkeypatch: pytest.MonkeyPatch): +async def _marked_messages( + messages: list[AllMessageValues], + points: list[CacheControlMessageInjectionPoint], + monkeypatch: pytest.MonkeyPatch, +) -> list[AllMessageValues]: monkeypatch.setenv("ANTHROPIC_API_KEY", "fake_anthropic_key") monkeypatch.setattr(litellm, "callbacks", [AnthropicCacheControlHook()]) client = AsyncHTTPHandler() @@ -6309,3 +6314,47 @@ async def test_anthropic_cache_control_hook_two_points_stay_two_breakpoints(monk "content": [{"type": "text", "text": "[System: Empty message content sanitised to satisfy protocol]"}], }, ] + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_skips_a_tool_reference_block(monkeypatch: pytest.MonkeyPatch): + """ + The tool_result conversion rebuilds a tool_reference from its type and tool_name + alone, so a marker written on one never reaches the provider. + """ + marked = await _marked_messages( + [ + {"role": "user", "content": "find a tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "tool_search", "arguments": "{}"}} + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [ + {"type": "text", "text": "found"}, + {"type": "tool_reference", "tool_name": "get_weather"}, + ], + }, + ], + [{"location": "message", "index": 2}], + monkeypatch, + ) + + assert marked[-1] == { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_1", + "content": [ + {"type": "text", "text": "found", "cache_control": {"type": "ephemeral"}}, + {"type": "tool_reference", "tool_name": "get_weather"}, + ], + } + ], + } From 24332d80ff693fa6f8497f1dc4909721fd2037ad Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Wed, 23 Sep 2026 16:02:51 +0700 Subject: [PATCH 06/10] test(anthropic): cover the guards that refuse a non-object block or message A bare string in a content list is skipped by the walk-back, so the request still goes out; a bare string in the message list is left to litellm's own validation. Removing either guard raises AttributeError out of the hook. --- .../test_anthropic_cache_control_hook.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 8ec4431eaf0..5d2687d89fd 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -6358,3 +6358,35 @@ async def test_anthropic_cache_control_hook_skips_a_tool_reference_block(monkeyp } ], } + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_skips_a_block_that_is_not_an_object(monkeypatch: pytest.MonkeyPatch): + """A caller can put a bare string in a content list; reading .type off it raises.""" + marked = await _marked_messages( + [ + {"role": "user", "content": "What is 2 + 2?"}, + {"role": "assistant", "content": ["The answer is 4."]}, + ], + [{"location": "message", "role": "assistant", "index": -1}], + monkeypatch, + ) + assert marked == [{"role": "user", "content": [{"type": "text", "text": "What is 2 + 2?"}]}] + + +@pytest.mark.asyncio +async def test_anthropic_cache_control_hook_leaves_a_message_that_is_not_an_object_to_litellm( + monkeypatch: pytest.MonkeyPatch, +): + """A bare string in the message list belongs to litellm's own validation, not to this hook.""" + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake_anthropic_key") + monkeypatch.setattr(litellm, "callbacks", [AnthropicCacheControlHook()]) + client = AsyncHTTPHandler() + with patch.object(client, "post", return_value=_anthropic_response_mock()): + with pytest.raises(litellm.APIConnectionError): + await litellm.acompletion( + model="anthropic/claude-sonnet-4-5", + messages=[{"role": "user", "content": "What is 2 + 2?"}, "The answer is 4."], + cache_control_injection_points=[{"location": "message", "index": -1}], + client=client, + ) From b3e5515c0a9fcca47651549906d420dea55b281b Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Thu, 1 Oct 2026 21:17:54 +0700 Subject: [PATCH 07/10] fix(anthropic): return the resolved indices as a tuple and type the narrowed message dict The resolver built three lists that nothing mutates, and the message a type guard narrowed to dict carried unknown key and value types into the block walk-back. Both sit under the repo's type-discipline and basedpyright budgets. --- .../integrations/anthropic_cache_control_hook.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 70cd0887442..12c404b8bf1 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -795,8 +795,9 @@ def _message_accepts_cache_control(message: object) -> bool: """Whether a marker written on this message reaches the provider.""" if not isinstance(message, dict): return False - on_a_tool_message: Final = message.get("role") == "tool" - content: Final = message.get("content") + fields: Final = cast(dict[str, object], message) # cast-ok: a runtime dict whose value types are not known here + on_a_tool_message: Final = fields.get("role") == "tool" + content: Final = fields.get("content") if isinstance(content, str): return content != "" or on_a_tool_message if isinstance(content, list): @@ -1180,7 +1181,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): @staticmethod def _resolve_target_indices( point: CacheControlMessageInjectionPoint, messages: list[AllMessageValues], answered_tool_ids: Container[str] - ) -> list[int]: + ) -> tuple[int, ...]: """Resolve which message indices an injection point targets.""" _targetted_index: Final[int | str | None] = point.get("index", None) targetted_index: int | None = None @@ -1207,7 +1208,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Case 1: Target by role alone if targetted_index is None: - return [] if targetted_role is None else [index for index in candidates if index in free] + return () if targetted_role is None else tuple(index for index in candidates if index in free) # Case 2: Target by index, counted within the role the point named position: Final = targetted_index + len(candidates) if targetted_index < 0 else targetted_index @@ -1218,10 +1219,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): len(candidates), position, ) - return [] + return () landing: Final = next((candidates[step] for step in range(position, -1, -1) if candidates[step] in free), None) - return [] if landing is None else [landing] + return () if landing is None else (landing,) @staticmethod def _count_cache_control_blocks(message: object, answered_tool_ids: Container[str] = frozenset()) -> int: From 4aab8e524bce70df6177064b0ed9ac30d979e1d5 Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Fri, 9 Oct 2026 02:31:27 +0700 Subject: [PATCH 08/10] fix(anthropic): type the block and role reads so the basedpyright budget stays flat on main --- .../anthropic_cache_control_hook.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 12c404b8bf1..0843c4817d8 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -773,7 +773,7 @@ def _message_dropping_created_cache_controls_between( ) -def _index_of_block_accepting_cache_control(content: list[object], on_a_tool_message: bool) -> int | None: +def _index_of_block_accepting_cache_control(content: Sequence[object], on_a_tool_message: bool) -> int | None: """Last block a marker written here reaches the provider on, searched from the end. An empty text block is replaced by a placeholder unless it sits on a tool message, @@ -783,14 +783,22 @@ def _index_of_block_accepting_cache_control(content: list[object], on_a_tool_mes block = content[index] if not isinstance(block, dict): continue - if block.get("type") in ANTHROPIC_BLOCK_TYPES_WITHOUT_CACHE_CONTROL: + fields = cast(dict[str, object], block) # cast-ok: a runtime dict whose value types are not known here + if fields.get("type") in ANTHROPIC_BLOCK_TYPES_WITHOUT_CACHE_CONTROL: continue - if block.get("type") == "text" and not block.get("text") and not on_a_tool_message: + if fields.get("type") == "text" and not fields.get("text") and not on_a_tool_message: continue return index return None +def _role_of(message: object) -> object: + """The ``role`` of a message dict, or None for anything else.""" + if not isinstance(message, dict): + return None + return cast(dict[str, object], message).get("role") # cast-ok: a runtime dict whose value types are not known here + + def _message_accepts_cache_control(message: object) -> bool: """Whether a marker written on this message reaches the provider.""" if not isinstance(message, dict): @@ -801,7 +809,8 @@ def _message_accepts_cache_control(message: object) -> bool: if isinstance(content, str): return content != "" or on_a_tool_message if isinstance(content, list): - return _index_of_block_accepting_cache_control(content, on_a_tool_message) is not None + blocks: Final = cast(Sequence[object], content) # cast-ok: a runtime list whose block types are not known here + return _index_of_block_accepting_cache_control(blocks, on_a_tool_message) is not None return False @@ -1197,7 +1206,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): candidates: Final = tuple( index for index, message in enumerate(messages) - if targetted_role is None or message.get("role") == targetted_role + if targetted_role is None or _role_of(message) == targetted_role ) free: Final = frozenset( index @@ -1277,8 +1286,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): message["cache_control"] = control # 2. list of objects - the last block that accepts a marker, per Anthropic spec elif isinstance(message_content, list): + blocks: Final = cast(Sequence[object], message_content) # cast-ok: TypedDict blocks read as objects target_index: Final = _index_of_block_accepting_cache_control( - message_content, on_a_tool_message=message.get("role") == "tool" + blocks, on_a_tool_message=_role_of(message) == "tool" ) if target_index is not None: message_content[target_index]["cache_control"] = control # pyright: ignore[reportGeneralTypeIssues] # loose runtime dict From 5dbc8de52b54d8115dd54f40ab60fef4f2d2abda Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Thu, 22 Oct 2026 21:23:34 +0700 Subject: [PATCH 09/10] fix(anthropic): skip empty text and tool_reference blocks in main's cache_control walk back main walks back past thinking and redacted_thinking blocks when it picks the block that carries a message's cache_control marker. An empty text block and a tool_reference block lose the marker too: the first is replaced by a placeholder before the request goes out, unless it sits on a tool message, and the tool_result conversion rebuilds the second from its type and tool_name alone. Extend _accepts_cache_control to refuse both, and drop this branch's own walk-back helper so one function decides where the marker goes. The thinking-block test now expects the redacted_thinking block the transform keeps since that change. --- .../anthropic_cache_control_hook.py | 56 +++++++------------ .../test_anthropic_cache_control_hook.py | 1 + 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 0843c4817d8..ddfbc81389e 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -84,12 +84,6 @@ _O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)") OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset( {"text", "image", "image_url", "file", "input_audio", "input_text", "input_image", "input_file"} ) -# Block types a marker never reaches the provider on. Anthropic accepts one on text, -# image, tool_use, tool_result and document, so a thinking block is refused there; a -# tool_reference is rebuilt from its type and tool_name alone by the tool_result -# conversion, which drops everything else. A list of refused types rather than -# accepted ones, so a block type this code has not been told about still takes one. -ANTHROPIC_BLOCK_TYPES_WITHOUT_CACHE_CONTROL: Final = frozenset({"thinking", "redacted_thinking", "tool_reference"}) OPENAI_API_HOST: Final = "api.openai.com" OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE") _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) @@ -773,25 +767,6 @@ def _message_dropping_created_cache_controls_between( ) -def _index_of_block_accepting_cache_control(content: Sequence[object], on_a_tool_message: bool) -> int | None: - """Last block a marker written here reaches the provider on, searched from the end. - - An empty text block is replaced by a placeholder unless it sits on a tool message, - where it goes out inside the tool_result. - """ - for index in range(len(content) - 1, -1, -1): - block = content[index] - if not isinstance(block, dict): - continue - fields = cast(dict[str, object], block) # cast-ok: a runtime dict whose value types are not known here - if fields.get("type") in ANTHROPIC_BLOCK_TYPES_WITHOUT_CACHE_CONTROL: - continue - if fields.get("type") == "text" and not fields.get("text") and not on_a_tool_message: - continue - return index - return None - - def _role_of(message: object) -> object: """The ``role`` of a message dict, or None for anything else.""" if not isinstance(message, dict): @@ -810,7 +785,7 @@ def _message_accepts_cache_control(message: object) -> bool: return content != "" or on_a_tool_message if isinstance(content, list): blocks: Final = cast(Sequence[object], content) # cast-ok: a runtime list whose block types are not known here - return _index_of_block_accepting_cache_control(blocks, on_a_tool_message) is not None + return _last_index_accepting_cache_control(blocks, on_a_tool_message) is not None return False @@ -824,18 +799,28 @@ EXTERNAL_BREAKPOINTS_STAMP: Final = "_litellm_external_breakpoints" CUSTOM_LLM_PROVIDER_STAMP: Final = "_litellm_custom_llm_provider" -def _accepts_cache_control(part: object) -> bool: +def _accepts_cache_control(part: object, on_a_tool_message: bool) -> bool: + # A tool_reference is rebuilt from its type and tool_name alone by the tool_result + # conversion, and an empty text block is replaced by a placeholder unless it sits on + # a tool message, where it goes out inside the tool_result: a marker on either is lost. match part: - case {"type": "thinking" | "redacted_thinking"}: + case {"type": "thinking" | "redacted_thinking" | "tool_reference"}: return False + case {"type": "text", "text": str(text)} if text: + return True + case {"type": "text"}: + return on_a_tool_message case dict(): return True case _: return False -def _last_index_accepting_cache_control(parts: Sequence[object]) -> int | None: - return next((index for index in reversed(range(len(parts))) if _accepts_cache_control(parts[index])), None) +def _last_index_accepting_cache_control(parts: Sequence[object], on_a_tool_message: bool) -> int | None: + return next( + (index for index in reversed(range(len(parts))) if _accepts_cache_control(parts[index], on_a_tool_message)), + None, + ) class AnthropicCacheControlHook(CustomPromptManagement): @@ -1284,14 +1269,13 @@ class AnthropicCacheControlHook(CustomPromptManagement): # 1. if string, insert cache control in the message if isinstance(message_content, str): message["cache_control"] = control - # 2. list of objects - the last block that accepts a marker, per Anthropic spec + # 2. list of objects - only apply to the last item Claude accepts a breakpoint on elif isinstance(message_content, list): - blocks: Final = cast(Sequence[object], message_content) # cast-ok: TypedDict blocks read as objects - target_index: Final = _index_of_block_accepting_cache_control( - blocks, on_a_tool_message=_role_of(message) == "tool" + markable_index: Final = _last_index_accepting_cache_control( + message_content, on_a_tool_message=_role_of(message) == "tool" ) - if target_index is not None: - message_content[target_index]["cache_control"] = control # pyright: ignore[reportGeneralTypeIssues] # loose runtime dict + if markable_index is not None: + message_content[markable_index]["cache_control"] = control # pyright: ignore[reportGeneralTypeIssues] # loose runtime dict return message @staticmethod diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 5d2687d89fd..42de6020c6e 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -5916,6 +5916,7 @@ async def test_anthropic_cache_control_hook_skips_a_thinking_block(monkeypatch: "content": [ {"type": "text", "text": "The answer is 4.", "cache_control": {"type": "ephemeral"}}, {"type": "thinking", "thinking": "Adding two and two.", "signature": "sig"}, + {"type": "redacted_thinking", "data": "redacted"}, ], } From 69117370f2575655f3c3979bbbdc51303ee749e2 Mon Sep 17 00:00:00 2001 From: Tan Nguyen Date: Wed, 4 Nov 2026 17:51:24 +0700 Subject: [PATCH 10/10] fix(anthropic): leave a message that is not an object to litellm's validation under main's breakpoint strip --- litellm/integrations/anthropic_cache_control_hook.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index ddfbc81389e..33b7ba86ca2 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -907,7 +907,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) ) processed_messages = copy.deepcopy( - [without_prompt_cache_breakpoints(message) for message in messages] + [ + without_prompt_cache_breakpoints(message) if isinstance(message, dict) else message # pyright: ignore[reportUnnecessaryIsInstance] # a caller can send a non-object message + for message in messages + ] if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider) else messages )