scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

fix: reserve cap slots for direct marks on /v1/messages when extra_body unmarks them

Browse source
This commit is contained in:
mateo-berri 2026-09-21 11:39:25 -07:00
parent 827d1c99a0
commit f567fe230e
2 changed files with 19 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -335,6 +335,22 @@ class AnthropicCacheControlHook(CustomPromptManagement):
)
return int(wire_cache_control is not None) + tool_blocks + envelope_blocks
@staticmethod
def count_external_cache_breakpoints_on_messages_route(
tools: Iterable[object] | None, cache_control: object, request_kwargs: object
) -> int:
"""The /v1/messages census before the route splits.
The native messages transforms drop the ``extra_body`` envelope while the
chat bridge merges it, so the cap reserves for whichever census is larger
rather than letting an envelope that unmarks a direct tool free a slot the
provider still counts.
"""
return max(
AnthropicCacheControlHook.count_external_cache_breakpoints(tools, cache_control),
AnthropicCacheControlHook.count_external_cache_breakpoints(tools, cache_control, request_kwargs),
)
@staticmethod
def _blocks_reserved_outside_messages(
remaining_points: Sequence[CacheControlInjectionPoint], external_breakpoints: int, openai_dialect: bool
@ -968,7 +984,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
system=system,
injection_points=injection_points,
openai_dialect=openai_dialect,
external_breakpoints=AnthropicCacheControlHook.count_external_cache_breakpoints(
external_breakpoints=AnthropicCacheControlHook.count_external_cache_breakpoints_on_messages_route(
tools, cache_control, kwargs
),
)

4
tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
View file

@ -2608,13 +2608,13 @@ class TestConfiguredInjectionPointsSurviveClientMarks:
"kwargs,tools,marked_turns,expected_system",
[
({"extra_body": {"tools": [MARKED_V1_TOOL]}}, [MARKED_V1_TOOL], 2, MARKED_SYSTEM),
({"extra_body": {"tools": [UNMARKED_V1_TOOL]}}, [MARKED_V1_TOOL], 3, MARKED_SYSTEM),
({"extra_body": {"tools": [UNMARKED_V1_TOOL]}}, [MARKED_V1_TOOL], 3, "sys"),
({"extra_body": {"tools": [MARKED_V1_TOOL]}}, [UNMARKED_V1_TOOL], 3, "sys"),
({"extra_body": {"cache_control": EPHEMERAL}, "cache_control": EPHEMERAL}, None, 2, MARKED_SYSTEM),
],
ids=["same_marked_tool_both_ways", "extra_body_unmarks", "extra_body_marks", "root_cache_control_both_ways"],
)
def test_v1_messages_cap_counts_extra_body_fields_in_place_of_the_direct_ones(
def test_v1_messages_cap_reserves_for_the_larger_of_direct_and_extra_body_marks(
self, kwargs, tools, marked_turns, expected_system
):
kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED), **copy.deepcopy(kwargs)}