mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-05 02:41:56 +00:00
fix: apply configured cache_control_injection_points beside client cache_control marks
Configured injection points were dropped whenever the request already carried a client-set cache_control anywhere, so an operator's rolling tail checkpoint silently never landed once a caller marked its own system prompt. Only the automatic defaults stand down now. Configured points skip a target the client already marked and stay under the provider's 4-block cap, counting the client's marks on messages, system, tools and the root cache_control first. The chat path carries the tool count as a stamp on the points because the prompt-management hook never receives tools. Fixes #40675
This commit is contained in:
mateo-berri
parent
6b8db68fb8
commit
ec59078ad9
3 changed files with 270 additions and 180 deletions
|
|
@ -121,6 +121,12 @@ def _carries_cache_breakpoint(block: object) -> bool:
|
|||
return isinstance(block, dict) and any(block.get(key) is not None for key in CACHE_BREAKPOINT_KEYS)
|
||||
|
||||
|
||||
def _tool_carries_cache_breakpoint(tool: object) -> bool:
|
||||
return _carries_cache_breakpoint(tool) or (
|
||||
isinstance(tool, dict) and _carries_cache_breakpoint(tool.get("function"))
|
||||
)
|
||||
|
||||
|
||||
def _accepts_prompt_cache_breakpoint(block: object) -> bool:
|
||||
return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
|
||||
|
||||
|
|
@ -131,6 +137,8 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
|
|||
# rather than spending them on a list that is still missing some of their targets.
|
||||
CARRY_UNMATCHED_MESSAGE_POINTS: Final = "_litellm_carry_unmatched_cache_control_points"
|
||||
|
||||
EXTERNAL_BREAKPOINTS_STAMP: Final = "_litellm_external_breakpoints"
|
||||
|
||||
|
||||
class AnthropicCacheControlHook(CustomPromptManagement):
|
||||
@staticmethod
|
||||
|
|
@ -205,10 +213,6 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
else:
|
||||
remaining_points.append(point)
|
||||
|
||||
# Non-message points (currently Bedrock tool_config) are handled in the
|
||||
# provider transform, where each tool_config point appends at most one
|
||||
# cachePoint to the tools. That block also counts toward Anthropic's
|
||||
# limit, so reserve a slot for it here to leave room.
|
||||
stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect")
|
||||
openai_dialect: Final = (
|
||||
stamped_dialect
|
||||
|
|
@ -233,8 +237,11 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
if carry_unmatched
|
||||
else tuple(message_points)
|
||||
)
|
||||
reserved_blocks: Final = (
|
||||
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
stamped_external: Final = injection_points[0].get(EXTERNAL_BREAKPOINTS_STAMP)
|
||||
reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages(
|
||||
remaining_points,
|
||||
stamped_external if isinstance(stamped_external, int) else 0,
|
||||
openai_dialect,
|
||||
)
|
||||
breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
|
||||
processed_messages = self._apply_message_injections(
|
||||
|
|
@ -251,14 +258,12 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
|
||||
# Points this pass did not place: non-message ones for the provider transform, and
|
||||
# the deferred role-targeted ones. Deferring is what reaches the Responses API's
|
||||
# `instructions`, which is only a system message once the bridge builds one. The
|
||||
# judged stamp is what makes it safe: the next pass must not re-judge points
|
||||
# against messages this pass already marked (see `_should_stand_down`).
|
||||
# `instructions`, which is only a system message once the bridge builds one. A later
|
||||
# pass re-applies them safely: a target that already carries a mark is skipped and
|
||||
# the census counts every mark on the wire, litellm's own included.
|
||||
carried_points: Final[Sequence[CacheControlInjectionPoint]] = (*remaining_points, *carried_message_points)
|
||||
if carried_points:
|
||||
non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(
|
||||
carried_points
|
||||
)
|
||||
non_default_params["cache_control_injection_points"] = list(carried_points)
|
||||
|
||||
return model, processed_messages, non_default_params
|
||||
|
||||
|
|
@ -293,6 +298,34 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
)
|
||||
return system_blocks + sum(AnthropicCacheControlHook._count_cache_control_blocks(msg) for msg in messages)
|
||||
|
||||
@staticmethod
|
||||
def count_external_cache_breakpoints(tools: Iterable[object] | None, cache_control: object = None) -> int:
|
||||
"""Client breakpoints outside messages and system that the provider cap still counts.
|
||||
|
||||
A tool carries its mark at the top level (Anthropic shape) or under ``function``
|
||||
(OpenAI shape); the Anthropic chat transform forwards both. A top-level
|
||||
``cache_control`` is Anthropic's automatic caching, which places one breakpoint
|
||||
of its own on top of the explicit ones.
|
||||
"""
|
||||
automatic_blocks: Final = 1 if cache_control is not None else 0
|
||||
tool_blocks: Final = sum(1 for tool in tools if _tool_carries_cache_breakpoint(tool)) if tools else 0
|
||||
return automatic_blocks + tool_blocks
|
||||
|
||||
@staticmethod
|
||||
def _blocks_reserved_outside_messages(
|
||||
remaining_points: Sequence[CacheControlInjectionPoint], external_breakpoints: int, openai_dialect: bool
|
||||
) -> int:
|
||||
"""Slots of the provider cap that the message census cannot see.
|
||||
|
||||
The client's breakpoints on tools and its automatic top-level ``cache_control``
|
||||
are already on the wire, and a ``tool_config`` point becomes one more cachePoint
|
||||
in the Bedrock converse transform. OpenAI's cap counts only its own block markers.
|
||||
"""
|
||||
if openai_dialect:
|
||||
return 0
|
||||
tool_config_blocks: Final = 1 if any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
return external_breakpoints + tool_config_blocks
|
||||
|
||||
@staticmethod
|
||||
def _apply_message_injections(
|
||||
points: Sequence[CacheControlMessageInjectionPoint],
|
||||
|
|
@ -473,11 +506,16 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
def apply_to_anthropic_messages_request(
|
||||
messages: list[dict],
|
||||
system: str | list | None,
|
||||
injection_points: list[CacheControlInjectionPoint],
|
||||
injection_points: Sequence[CacheControlInjectionPoint],
|
||||
openai_dialect: bool = False,
|
||||
external_breakpoints: int = 0,
|
||||
) -> tuple[list[dict], str | list | None, list[CacheControlInjectionPoint]]:
|
||||
"""Apply cache control injection for the Anthropic-native v1/messages endpoint.
|
||||
|
||||
``external_breakpoints`` is the client's breakpoint count outside ``messages`` and
|
||||
``system`` (see ``count_external_cache_breakpoints``); it shrinks the budget so
|
||||
the request never exceeds the provider cap.
|
||||
|
||||
Returns (messages, system, remaining_non_message_points).
|
||||
"""
|
||||
if not injection_points:
|
||||
|
|
@ -500,8 +538,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
else:
|
||||
remaining_points.append(point)
|
||||
|
||||
reserved_blocks: Final = (
|
||||
1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
|
||||
reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages(
|
||||
remaining_points, external_breakpoints, openai_dialect
|
||||
)
|
||||
max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
|
||||
|
||||
|
|
@ -556,30 +594,26 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
return ChatCompletionCachedContent(type="ephemeral")
|
||||
|
||||
@staticmethod
|
||||
def _stamped_as_judged(points: Sequence[CacheControlInjectionPoint]) -> Sequence[Mapping[str, object]]:
|
||||
"""Mark written-back points as having passed the client cache_control judgment.
|
||||
|
||||
Builds copies because config-owned point dicts are shared across
|
||||
requests; mutating them would leak the stamp into future requests.
|
||||
"""
|
||||
return AnthropicCacheControlHook._stamped(points, "_litellm_judged", True)
|
||||
|
||||
@staticmethod
|
||||
def _judged_configured_points(
|
||||
def _stamped_for_prompt_hook(
|
||||
points: Sequence[CacheControlInjectionPoint],
|
||||
messages: list[AllMessageValues],
|
||||
tools: list[object] | None,
|
||||
cache_control: object,
|
||||
external_breakpoints: int,
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: object,
|
||||
prompt_cache_options: object,
|
||||
) -> Sequence[Mapping[str, object]] | None:
|
||||
if AnthropicCacheControlHook._should_stand_down(points, messages, None, tools, cache_control):
|
||||
return None
|
||||
return AnthropicCacheControlHook._stamped_with_dialect(
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
"""Carry onto the points what the prompt-management hook never receives.
|
||||
|
||||
The hook sees neither the tools nor the request kwargs, so the target dialect
|
||||
and the client's breakpoint count outside the message list ride on the points.
|
||||
Builds copies because config-owned point dicts are shared across requests.
|
||||
"""
|
||||
with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect(
|
||||
points, model, custom_llm_provider, api_base, prompt_cache_options
|
||||
)
|
||||
if external_breakpoints == 0:
|
||||
return with_dialect
|
||||
return AnthropicCacheControlHook._stamped(with_dialect, EXTERNAL_BREAKPOINTS_STAMP, external_breakpoints)
|
||||
|
||||
@staticmethod
|
||||
def _stamped_with_dialect(
|
||||
|
|
@ -600,32 +634,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _stamped(
|
||||
points: Sequence[CacheControlInjectionPoint], key: str, value: object
|
||||
) -> Sequence[Mapping[str, object]]:
|
||||
def _stamped(points: Sequence[Mapping[str, object]], key: str, value: object) -> Sequence[Mapping[str, object]]:
|
||||
return [{**point, key: value} for point in points]
|
||||
|
||||
@staticmethod
|
||||
def _should_stand_down(
|
||||
points: Sequence[CacheControlInjectionPoint],
|
||||
messages: list[AllMessageValues],
|
||||
system: str | list | None,
|
||||
tools: list | None,
|
||||
cache_control: object = None,
|
||||
) -> bool:
|
||||
"""Whether configured injection points must yield to client-set cache_control.
|
||||
|
||||
Points that a prior pass over this request already judged and wrote
|
||||
back carry the internal judged stamp; any re-entry (acompletion
|
||||
re-entering completion, the async-to-sync /v1/messages dispatch,
|
||||
interceptor sub-calls reusing the request kwargs) must not re-judge
|
||||
them, because by then the messages carry litellm's own injected marks
|
||||
and the judgment would misread those as client breakpoints.
|
||||
"""
|
||||
if all(point.get("_litellm_judged") for point in points):
|
||||
return False
|
||||
return AnthropicCacheControlHook._request_has_cache_control(messages, system, tools, cache_control)
|
||||
|
||||
@staticmethod
|
||||
def _request_has_cache_control(
|
||||
messages: list[AllMessageValues],
|
||||
|
|
@ -635,28 +646,15 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
) -> bool:
|
||||
"""Return True if the request already carries any client-supplied cache_control.
|
||||
|
||||
When the client (e.g. Claude Code) already marks its own breakpoints we
|
||||
stand down entirely rather than add more, per the auto-caching contract.
|
||||
Tools count: they are a breakpoint the client can mark, they count toward
|
||||
the provider's four-block limit, and caching only the tool definitions is
|
||||
a common pattern, so injecting alongside them can exceed the cap. Tools
|
||||
carry the mark either at the top level (Anthropic shape) or nested under
|
||||
``function`` (OpenAI shape); the Anthropic chat transform accepts both.
|
||||
Only the automatic defaults stand down on it: a client that marks its own
|
||||
breakpoints (Claude Code does) has a caching strategy the defaults would
|
||||
clash with. Configured injection points are an explicit instruction and are
|
||||
applied alongside the client's marks, bounded by the provider cap.
|
||||
"""
|
||||
if cache_control is not None:
|
||||
return True
|
||||
if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0:
|
||||
return True
|
||||
if tools is not None:
|
||||
return any(
|
||||
isinstance(tool, dict)
|
||||
and (
|
||||
tool.get("cache_control") is not None
|
||||
or (isinstance(tool.get("function"), dict) and tool["function"].get("cache_control") is not None)
|
||||
)
|
||||
for tool in tools
|
||||
)
|
||||
return False
|
||||
return (
|
||||
AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
|
||||
+ AnthropicCacheControlHook.count_external_cache_breakpoints(tools, cache_control)
|
||||
) > 0
|
||||
|
||||
@staticmethod
|
||||
def get_default_injection_points(
|
||||
|
|
@ -779,31 +777,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
) -> None:
|
||||
"""For /chat/completions: resolve the injection points the request should carry.
|
||||
|
||||
Configured injection points win over the automatic defaults, but stand
|
||||
down entirely when the client already marked its own cache_control
|
||||
breakpoints (messages or tools): injecting alongside them clashes with
|
||||
the client's caching strategy and can exceed the provider's four-block
|
||||
limit. The judgment happens once per request; points a prior pass
|
||||
wrote back carry the judged stamp and are never re-judged (see
|
||||
``_should_stand_down``). Seeding the param lets the existing
|
||||
prompt-management gate and the AnthropicCacheControlHook run
|
||||
unchanged.
|
||||
Configured injection points win over the automatic defaults and are applied
|
||||
even when the client marked its own cache_control elsewhere in the request;
|
||||
the provider's four-block cap bounds them, counting the client's marks on
|
||||
messages, tools and the top-level ``cache_control``. Only the defaults stand
|
||||
down on client marks. Seeding the param lets the existing prompt-management
|
||||
gate and the AnthropicCacheControlHook run unchanged.
|
||||
"""
|
||||
if non_default_params.get("cache_control_injection_points"):
|
||||
judged: Final = AnthropicCacheControlHook._judged_configured_points(
|
||||
non_default_params["cache_control_injection_points"],
|
||||
messages,
|
||||
tools,
|
||||
non_default_params.get("cache_control"),
|
||||
configured: Final = non_default_params.get("cache_control_injection_points")
|
||||
if configured:
|
||||
non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_for_prompt_hook(
|
||||
configured,
|
||||
AnthropicCacheControlHook.count_external_cache_breakpoints(
|
||||
tools, non_default_params.get("cache_control")
|
||||
),
|
||||
model,
|
||||
custom_llm_provider,
|
||||
api_base,
|
||||
non_default_params.get("prompt_cache_options"),
|
||||
)
|
||||
if judged is None:
|
||||
non_default_params.pop("cache_control_injection_points")
|
||||
else:
|
||||
non_default_params["cache_control_injection_points"] = judged
|
||||
return
|
||||
points: Final = AnthropicCacheControlHook.get_default_injection_points(
|
||||
messages=messages,
|
||||
|
|
@ -904,15 +896,14 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
) -> tuple[list[dict], str | list | None]:
|
||||
"""Extract cache_control_injection_points from kwargs and apply if present.
|
||||
|
||||
Configured points stand down entirely when the client already marked
|
||||
its own cache_control breakpoints anywhere in the request. The
|
||||
judgment happens once per request; points a prior pass wrote back
|
||||
carry the judged stamp and are never re-judged (see
|
||||
``_should_stand_down``). When none are configured but
|
||||
Configured points are applied even when the client marked its own
|
||||
cache_control elsewhere in the request, bounded by the provider cap,
|
||||
which counts the client's marks on messages, system, tools and the
|
||||
top-level ``cache_control``. When none are configured but
|
||||
``litellm.enable_anthropic_prompt_caching`` or the per-request
|
||||
``enable_prompt_caching`` kwarg (stamped from key metadata) is on,
|
||||
synthesize default breakpoints for the native /v1/messages path. Pops
|
||||
both keys from kwargs;
|
||||
synthesize default breakpoints for the native /v1/messages path; those
|
||||
defaults alone stand down on client marks. Pops both keys from kwargs;
|
||||
if remaining (non-message) points exist they are written back so
|
||||
downstream transforms can handle them.
|
||||
"""
|
||||
|
|
@ -924,13 +915,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
configured: Final = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list
|
||||
list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None)
|
||||
)
|
||||
if configured and AnthropicCacheControlHook._should_stand_down(
|
||||
configured, typed_messages, system, tools, cache_control
|
||||
):
|
||||
return messages, system
|
||||
injection_points: list[CacheControlInjectionPoint] = configured or []
|
||||
if not injection_points and model is not None:
|
||||
injection_points = AnthropicCacheControlHook.get_default_injection_points(
|
||||
injection_points: Final[Sequence[CacheControlInjectionPoint]] = configured or (
|
||||
AnthropicCacheControlHook.get_default_injection_points(
|
||||
messages=typed_messages,
|
||||
system=system,
|
||||
tools=tools,
|
||||
|
|
@ -940,6 +926,9 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
cache_control=cache_control,
|
||||
request_kwargs=kwargs,
|
||||
)
|
||||
if model is not None
|
||||
else ()
|
||||
)
|
||||
if not injection_points:
|
||||
return messages, system
|
||||
|
||||
|
|
@ -952,6 +941,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
system=system,
|
||||
injection_points=injection_points,
|
||||
openai_dialect=openai_dialect,
|
||||
external_breakpoints=AnthropicCacheControlHook.count_external_cache_breakpoints(tools, cache_control),
|
||||
)
|
||||
breakpoints_added: Final = (
|
||||
AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before
|
||||
|
|
@ -960,7 +950,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
if openai_dialect and breakpoints_added > 0:
|
||||
kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
|
||||
if remaining:
|
||||
kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)
|
||||
kwargs["cache_control_injection_points"] = remaining
|
||||
return messages, system
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ class CacheControlMessageInjectionPoint(TypedDict):
|
|||
role: Literal["user", "system", "assistant"] | None # Optional: target by role (user, system, assistant)
|
||||
index: int | str | None # Optional: target by specific index
|
||||
control: ChatCompletionCachedContent | None
|
||||
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
|
||||
_litellm_openai_dialect: NotRequired[ReadOnly[bool]]
|
||||
_litellm_external_breakpoints: NotRequired[ReadOnly[int]]
|
||||
|
||||
|
||||
class CacheControlToolConfigInjectionPoint(TypedDict):
|
||||
|
|
@ -26,8 +26,8 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
|
|||
|
||||
location: Literal["tool_config"]
|
||||
control: ChatCompletionCachedContent | None
|
||||
_litellm_judged: NotRequired[bool] # Internal: written back by litellm once the client cache_control judgment ran
|
||||
_litellm_openai_dialect: NotRequired[ReadOnly[bool]]
|
||||
_litellm_external_breakpoints: NotRequired[ReadOnly[int]]
|
||||
|
||||
|
||||
CacheControlInjectionPoint = CacheControlMessageInjectionPoint | CacheControlToolConfigInjectionPoint
|
||||
|
|
|
|||
|
|
@ -1276,11 +1276,7 @@ def test_cache_control_hook_reserves_slot_for_tool_config_point():
|
|||
)
|
||||
|
||||
assert _count_cache_control(processed) == 3
|
||||
# The tool_config point is passed through for the provider transform,
|
||||
# stamped so re-entries never re-judge it against litellm's own marks.
|
||||
assert non_default_params["cache_control_injection_points"] == [
|
||||
{"location": "tool_config", "_litellm_judged": True}
|
||||
]
|
||||
assert non_default_params["cache_control_injection_points"] == [{"location": "tool_config"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -2085,13 +2081,17 @@ class TestPerKeyEnablePromptCaching:
|
|||
assert result_msgs == messages
|
||||
|
||||
|
||||
class TestConfiguredInjectionPointsStandDown:
|
||||
"""Configured cache_control_injection_points must stand down entirely when the
|
||||
client already set its own cache_control anywhere in the request (LIT-4582);
|
||||
injecting alongside client breakpoints clashes with the client's caching
|
||||
strategy and can push the request past Anthropic's four-block limit."""
|
||||
class TestConfiguredInjectionPointsSurviveClientMarks:
|
||||
"""Configured cache_control_injection_points are an explicit instruction, so they
|
||||
apply alongside the client's own cache_control marks (LIT-7586, #40675) instead of
|
||||
standing down on them. What bounds them is Anthropic's four-block cap, which has to
|
||||
count the client's marks on messages, system, tools and the root ``cache_control``
|
||||
(LIT-4582: a client-marked tool the cap could not see produced "Found 5" 400s).
|
||||
Only the automatic defaults stand down on client marks."""
|
||||
|
||||
CONFIGURED = [{"location": "message", "role": "system"}]
|
||||
TAIL_POINT = [{"location": "message", "index": -1}]
|
||||
EPHEMERAL = {"type": "ephemeral"}
|
||||
|
||||
CLEAN_MESSAGES: List[AllMessageValues] = [
|
||||
{"role": "system", "content": "sys"},
|
||||
|
|
@ -2105,6 +2105,23 @@ class TestConfiguredInjectionPointsStandDown:
|
|||
|
||||
V1_MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
|
||||
MARKED_TOOL_TOP_LEVEL = {
|
||||
"type": "function",
|
||||
"function": {"name": "t", "parameters": {}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
MARKED_TOOL_NESTED = {"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}}
|
||||
UNMARKED_TOOL = {"type": "function", "function": {"name": "t", "parameters": {}}}
|
||||
MARKED_V1_TOOL = {"name": "t", "input_schema": {}, "cache_control": {"type": "ephemeral"}}
|
||||
UNMARKED_V1_TOOL = {"name": "t", "input_schema": {}}
|
||||
|
||||
@staticmethod
|
||||
def _marked_user_turns(count):
|
||||
return [
|
||||
{"role": "user", "content": [{"type": "text", "text": f"turn {i}", "cache_control": {"type": "ephemeral"}}]}
|
||||
for i in range(count)
|
||||
]
|
||||
|
||||
def _seed(self, params, messages, tools=None):
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
|
|
@ -2114,6 +2131,17 @@ class TestConfiguredInjectionPointsStandDown:
|
|||
tools=tools,
|
||||
)
|
||||
|
||||
def _chat(self, params, messages):
|
||||
_, processed, _ = AnthropicCacheControlHook().get_chat_completion_prompt(
|
||||
model="claude-sonnet-4-5",
|
||||
messages=messages,
|
||||
non_default_params=params,
|
||||
prompt_id=None,
|
||||
prompt_variables=None,
|
||||
dynamic_callback_params={},
|
||||
)
|
||||
return processed
|
||||
|
||||
def _inject(self, messages, kwargs, system="sys", tools=None):
|
||||
return AnthropicCacheControlHook.maybe_inject_cache_control(
|
||||
messages,
|
||||
|
|
@ -2124,23 +2152,64 @@ class TestConfiguredInjectionPointsStandDown:
|
|||
tools=tools,
|
||||
)
|
||||
|
||||
def test_configured_points_dropped_when_messages_carry_cache_control(self):
|
||||
def test_chat_tail_point_applies_when_client_marked_the_system_block(self):
|
||||
"""The issue's shape: the client caches its system prompt, the deployment is
|
||||
configured to cache the trailing turn, and both marks must reach the provider."""
|
||||
messages: List[AllMessageValues] = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": "history"},
|
||||
{"role": "assistant", "content": "reply"},
|
||||
{"role": "user", "content": "question"},
|
||||
]
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.TAIL_POINT)}
|
||||
self._seed(params, messages)
|
||||
processed = self._chat(params, messages)
|
||||
assert processed[0] == messages[0]
|
||||
assert processed[-1] == {"role": "user", "content": "question", "cache_control": self.EPHEMERAL}
|
||||
assert _count_cache_control(processed) == 2
|
||||
|
||||
def test_chat_configured_points_apply_when_messages_carry_cache_control(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
|
||||
self._seed(params, copy.deepcopy(self.MARKED_MESSAGES))
|
||||
assert "cache_control_injection_points" not in params
|
||||
processed = self._chat(params, copy.deepcopy(self.MARKED_MESSAGES))
|
||||
assert processed[0] == {"role": "system", "content": "sys", "cache_control": self.EPHEMERAL}
|
||||
assert processed[1] == self.MARKED_MESSAGES[1]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
[
|
||||
{"type": "function", "function": {"name": "t", "parameters": {}}, "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "function", "function": {"name": "t", "parameters": {}, "cache_control": {"type": "ephemeral"}}},
|
||||
],
|
||||
ids=["top_level", "nested_in_function"],
|
||||
"tool", [MARKED_TOOL_TOP_LEVEL, MARKED_TOOL_NESTED], ids=["top_level", "nested_in_function"]
|
||||
)
|
||||
def test_configured_points_dropped_when_tools_carry_cache_control(self, tool):
|
||||
def test_chat_configured_points_apply_when_tools_carry_cache_control(self, tool):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
|
||||
self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES), tools=[tool])
|
||||
assert "cache_control_injection_points" not in params
|
||||
processed = self._chat(params, copy.deepcopy(self.CLEAN_MESSAGES))
|
||||
assert processed[0] == {"role": "system", "content": "sys", "cache_control": self.EPHEMERAL}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool,injected",
|
||||
[(MARKED_TOOL_TOP_LEVEL, 0), (MARKED_TOOL_NESTED, 0), (UNMARKED_TOOL, 1)],
|
||||
ids=["marked_top_level", "marked_nested_in_function", "unmarked"],
|
||||
)
|
||||
def test_chat_cap_counts_client_marked_tools(self, tool, injected):
|
||||
"""LIT-4582 regression: the prompt-management hook never sees the tools, so the
|
||||
seeding pass has to carry the client's tool marks into the cap or a configured
|
||||
point lands as a fifth block."""
|
||||
messages = [{"role": "system", "content": "sys"}, *self._marked_user_turns(3)]
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
|
||||
self._seed(params, copy.deepcopy(messages), tools=[tool])
|
||||
processed = self._chat(params, copy.deepcopy(messages))
|
||||
assert _count_cache_control(processed) == 3 + injected
|
||||
|
||||
@pytest.mark.parametrize("marked_turns,injected", [(2, 1), (3, 0)])
|
||||
def test_chat_root_cache_control_reserves_a_slot(self, marked_turns, injected):
|
||||
"""Anthropic's automatic caching (a top-level ``cache_control``) places one
|
||||
breakpoint of its own, so it counts toward the cap like a client mark."""
|
||||
messages = [{"role": "system", "content": "sys"}, *self._marked_user_turns(marked_turns)]
|
||||
root_cache_control = {"type": "ephemeral"}
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED), "cache_control": root_cache_control}
|
||||
self._seed(params, copy.deepcopy(messages))
|
||||
processed = self._chat(params, copy.deepcopy(messages))
|
||||
assert _count_cache_control(processed) == marked_turns + injected
|
||||
assert params["cache_control"] is root_cache_control
|
||||
|
||||
def test_configured_points_kept_when_request_is_unmarked(self):
|
||||
configured = copy.deepcopy(self.CONFIGURED)
|
||||
|
|
@ -2148,60 +2217,68 @@ class TestConfiguredInjectionPointsStandDown:
|
|||
self._seed(params, copy.deepcopy(self.CLEAN_MESSAGES))
|
||||
assert params["cache_control_injection_points"] is configured
|
||||
|
||||
def test_judged_remainder_survives_reentry_despite_injected_marks(self):
|
||||
"""acompletion() re-enters completion() after injection ran, with only the
|
||||
stamped non-message points written back; the re-entry must not misread
|
||||
litellm's own marks as client ones and drop that remainder."""
|
||||
remainder = [{"location": "tool_config", "_litellm_judged": True}]
|
||||
params = {"cache_control_injection_points": remainder}
|
||||
self._seed(params, copy.deepcopy(self.MARKED_MESSAGES))
|
||||
assert params["cache_control_injection_points"] is remainder
|
||||
def test_chat_reentry_over_injected_messages_adds_no_duplicate_marks(self):
|
||||
"""acompletion() re-enters completion() and interceptor sub-calls reuse the
|
||||
request kwargs, so the same configured points meet messages that already carry
|
||||
litellm's own marks; the second pass must leave them as they are."""
|
||||
points = [{"location": "message", "role": "system"}, {"location": "tool_config"}]
|
||||
first_params = {"cache_control_injection_points": copy.deepcopy(points)}
|
||||
self._seed(first_params, copy.deepcopy(self.MARKED_MESSAGES))
|
||||
first = self._chat(first_params, copy.deepcopy(self.MARKED_MESSAGES))
|
||||
assert _count_cache_control(first) == 2
|
||||
assert first_params["cache_control_injection_points"] == [{"location": "tool_config"}]
|
||||
|
||||
def test_v1_messages_stand_down_when_content_block_marked(self):
|
||||
second_params = {"cache_control_injection_points": copy.deepcopy(points)}
|
||||
self._seed(second_params, copy.deepcopy(first))
|
||||
second = self._chat(second_params, copy.deepcopy(first))
|
||||
assert second == first
|
||||
assert second_params["cache_control_injection_points"] == [{"location": "tool_config"}]
|
||||
|
||||
def test_v1_messages_configured_point_applies_when_content_block_marked(self):
|
||||
messages = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}
|
||||
]
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
|
||||
result_msgs, result_sys = self._inject(copy.deepcopy(messages), kwargs)
|
||||
assert result_msgs == messages
|
||||
assert result_sys == "sys"
|
||||
assert result_sys == [{"type": "text", "text": "sys", "cache_control": self.EPHEMERAL}]
|
||||
assert "cache_control_injection_points" not in kwargs
|
||||
|
||||
def test_v1_messages_stand_down_when_system_block_marked(self):
|
||||
"""A configured point targeting a message must not fire when the client
|
||||
marked the system prompt; the old behavior injected into the message
|
||||
because only the exact targeted position was guarded."""
|
||||
def test_v1_messages_tail_point_applies_when_system_block_marked(self):
|
||||
system = [{"type": "text", "text": "s", "cache_control": {"type": "ephemeral"}}]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "role": "user"}]}
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.TAIL_POINT)}
|
||||
result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, system=system)
|
||||
assert result_msgs == self.V1_MESSAGES
|
||||
assert result_msgs == [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": self.EPHEMERAL}]}]
|
||||
assert result_sys == system
|
||||
assert "cache_control_injection_points" not in kwargs
|
||||
|
||||
def test_v1_messages_stand_down_when_tools_marked(self):
|
||||
tools = [{"name": "t", "input_schema": {}, "cache_control": {"type": "ephemeral"}}]
|
||||
def test_v1_messages_configured_point_applies_when_tools_marked(self):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
|
||||
result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, tools=tools)
|
||||
result_msgs, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs, tools=[self.MARKED_V1_TOOL])
|
||||
assert result_msgs == self.V1_MESSAGES
|
||||
assert result_sys == "sys"
|
||||
assert "cache_control_injection_points" not in kwargs
|
||||
assert result_sys == [{"type": "text", "text": "sys", "cache_control": self.EPHEMERAL}]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool,expected_system",
|
||||
[
|
||||
(MARKED_V1_TOOL, "sys"),
|
||||
(UNMARKED_V1_TOOL, [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]),
|
||||
],
|
||||
ids=["marked", "unmarked"],
|
||||
)
|
||||
def test_v1_messages_cap_counts_client_marked_tools(self, tool, expected_system):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
|
||||
_, result_sys = self._inject(self._marked_user_turns(3), kwargs, tools=[tool])
|
||||
assert result_sys == expected_system
|
||||
|
||||
def test_v1_messages_configured_points_apply_when_unmarked(self):
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.CONFIGURED)}
|
||||
_, result_sys = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs)
|
||||
assert result_sys == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"configured",
|
||||
[None, CONFIGURED],
|
||||
ids=["automatic_defaults", "configured_points"],
|
||||
)
|
||||
def test_v1_messages_stands_down_for_root_cache_control(self, monkeypatch, configured):
|
||||
def test_v1_messages_automatic_defaults_stand_down_for_root_cache_control(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True)
|
||||
root_cache_control = {"type": "ephemeral"}
|
||||
kwargs = {"cache_control": root_cache_control, "litellm_metadata": {}}
|
||||
if configured is not None:
|
||||
kwargs["cache_control_injection_points"] = copy.deepcopy(configured)
|
||||
|
||||
result_messages, result_system = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs)
|
||||
|
||||
|
|
@ -2210,17 +2287,33 @@ class TestConfiguredInjectionPointsStandDown:
|
|||
assert kwargs["cache_control"] is root_cache_control
|
||||
assert "litellm_gateway_injected_cache" not in kwargs["litellm_metadata"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"marked_turns,expected_system",
|
||||
[(2, [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]), (3, "sys")],
|
||||
)
|
||||
def test_v1_messages_configured_points_apply_with_root_cache_control_reserving_a_slot(
|
||||
self, marked_turns, expected_system
|
||||
):
|
||||
root_cache_control = {"type": "ephemeral"}
|
||||
kwargs = {
|
||||
"cache_control": root_cache_control,
|
||||
"cache_control_injection_points": copy.deepcopy(self.CONFIGURED),
|
||||
}
|
||||
_, result_system = self._inject(self._marked_user_turns(marked_turns), kwargs)
|
||||
assert result_system == expected_system
|
||||
assert kwargs["cache_control"] is root_cache_control
|
||||
|
||||
def test_v1_messages_reentry_flow_preserves_tool_config_remainder(self):
|
||||
"""The advisor interceptor re-enters anthropic_messages() with the outer
|
||||
request's kwargs and post-injection messages. The first pass applies the
|
||||
message point and writes back a stamped tool_config remainder; the
|
||||
re-entry must keep that remainder even though the messages and system
|
||||
now carry litellm's own marks."""
|
||||
message point and writes back the tool_config remainder; the re-entry must
|
||||
keep that remainder and add no mark even though the messages and system
|
||||
now carry litellm's own."""
|
||||
points = [{"location": "message", "role": "system"}, {"location": "tool_config"}]
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(points)}
|
||||
msgs1, sys1 = self._inject(copy.deepcopy(self.V1_MESSAGES), kwargs)
|
||||
assert sys1[0]["cache_control"] == {"type": "ephemeral"}
|
||||
expected_remainder = [{"location": "tool_config", "_litellm_judged": True}]
|
||||
expected_remainder = [{"location": "tool_config"}]
|
||||
assert kwargs["cache_control_injection_points"] == expected_remainder
|
||||
|
||||
msgs2, sys2 = self._inject(msgs1, kwargs, system=sys1)
|
||||
|
|
@ -2459,22 +2552,22 @@ class TestOpenAIPromptCacheBreakpoint:
|
|||
assert system == [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]
|
||||
assert kwargs == {}
|
||||
|
||||
def test_v1_messages_client_content_breakpoint_makes_configured_points_stand_down(self):
|
||||
def test_v1_messages_configured_points_apply_beside_client_content_breakpoint(self):
|
||||
messages = [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}]
|
||||
kwargs = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
result, system = self._inject(messages, "sys", kwargs)
|
||||
assert result == messages
|
||||
assert system == "sys"
|
||||
assert kwargs == {}
|
||||
assert system == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
assert kwargs == {"prompt_cache_options": self.EXPLICIT}
|
||||
|
||||
def test_v1_messages_client_system_breakpoint_makes_configured_points_stand_down(self):
|
||||
def test_v1_messages_tail_point_applies_beside_client_system_breakpoint(self):
|
||||
system = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
messages = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
kwargs = {"cache_control_injection_points": [{"location": "message", "index": -1}]}
|
||||
result, result_system = self._inject(messages, system, kwargs)
|
||||
assert result == messages
|
||||
assert result == [{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]}]
|
||||
assert result_system == system
|
||||
assert kwargs == {}
|
||||
assert kwargs == {"prompt_cache_options": self.EXPLICIT}
|
||||
|
||||
def test_chat_system_string_wrapped_with_block_breakpoint(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
|
|
@ -2538,18 +2631,25 @@ class TestOpenAIPromptCacheBreakpoint:
|
|||
assert processed[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}}
|
||||
assert params == {}
|
||||
|
||||
def test_chat_client_breakpoint_makes_seeded_points_stand_down(self):
|
||||
def test_chat_seeded_points_apply_beside_client_breakpoint(self):
|
||||
params = {"cache_control_injection_points": copy.deepcopy(self.SYSTEM_POINT)}
|
||||
messages = [
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]},
|
||||
]
|
||||
AnthropicCacheControlHook.maybe_seed_default_injection_points(
|
||||
non_default_params=params,
|
||||
messages=[
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi", "prompt_cache_breakpoint": self.EXPLICIT}]},
|
||||
],
|
||||
messages=messages,
|
||||
model="openai/gpt-5.6",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
assert params == {}
|
||||
assert params["cache_control_injection_points"] == [
|
||||
{"location": "message", "role": "system", "_litellm_openai_dialect": True}
|
||||
]
|
||||
_, processed, _ = self._chat(messages, params)
|
||||
assert processed[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": self.EXPLICIT}]
|
||||
assert processed[1] == messages[1]
|
||||
assert params["prompt_cache_options"] == self.EXPLICIT
|
||||
|
||||
def test_cap_counts_client_breakpoints_of_both_kinds(self):
|
||||
messages = [
|
||||
|
|
@ -3143,7 +3243,7 @@ class TestRecordGatewayInjection:
|
|||
assert kwargs["litellm_metadata"][self.KEY] == self.DEPLOYMENT
|
||||
|
||||
def test_configured_points_skipping_a_marked_target_record_nothing(self):
|
||||
"""Configured injection stands down on client breakpoints, so no marker lands."""
|
||||
"""A configured point whose target the client already marked places nothing, so no marker lands."""
|
||||
kwargs: dict = {
|
||||
"litellm_metadata": {},
|
||||
"cache_control_injection_points": [{"location": "message", "role": "system", "index": None}],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue