scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 86

fix(openai_like): scope cache_control normalization to Messages API locations

Browse source 
Rewrite the sanitizer without recursion (the code-quality gate rejects new
recursive functions) and only touch cache_control where the Messages API
defines it: the request, system blocks, tools, message content blocks, and
tool_result content. Application data such as tool_use.input and tool
input_schema is left untouched even when it contains a cache_control key
This commit is contained in:
mateo-berri 2026-08-29 14:07:05 -07:00
parent f4b5449c6a
commit 0baf376efd
2 changed files with 124 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
litellm/llms/anthropic/common_utils.py
View file

@ -1302,37 +1302,83 @@ def flatten_unencrypted_web_search_results_in_anthropic_messages( # mutable-ok:
return [_flatten_web_search_results_in_message(m) for m in messages] # mutable-ok: JSON wire format
def _normalized_cache_control(cache_control: dict) -> dict: # mutable-ok: as sibling sanitizers
def _normalized_cache_control(cache_control: object) -> dict[str, str] | None: # mutable-ok: JSON wire format
if not isinstance(cache_control, Mapping):
return None
cache_type: Final = cache_control.get("type")
return {"type": cache_type if isinstance(cache_type, str) else "ephemeral"} # mutable-ok: JSON wire format
def _normalize_cache_control_value(value: object) -> object:
if isinstance(value, dict):
return normalize_cache_control_in_anthropic_payload(value)
if isinstance(value, list):
return [_normalize_cache_control_value(item) for item in value] # mutable-ok: JSON wire format
return value
def _with_portable_cache_control(block: Mapping[str, object]) -> dict[str, object]: # mutable-ok: JSON wire format
if "cache_control" not in block:
return dict(block) # mutable-ok: JSON wire format
normalized: Final = _normalized_cache_control(block["cache_control"])
rest: Final = {key: value for key, value in block.items() if key != "cache_control"} # mutable-ok: JSON wire format
return rest if normalized is None else {**rest, "cache_control": normalized} # mutable-ok: JSON wire format
def normalize_cache_control_in_anthropic_payload(payload: dict) -> dict: # mutable-ok: as sibling sanitizers
def _with_portable_cache_control_in_blocks(blocks: object) -> object:
if isinstance(blocks, str) or not isinstance(blocks, Sequence):
return blocks
return [ # mutable-ok: JSON wire format
_with_portable_cache_control(block) if isinstance(block, Mapping) else block for block in blocks
]
def _with_portable_cache_control_in_content_block(block: object) -> object:
if not isinstance(block, Mapping):
return block
portable: Final = _with_portable_cache_control(block)
if portable.get("type") != "tool_result" or "content" not in portable:
return portable
return { # mutable-ok: JSON wire format
**portable,
"content": _with_portable_cache_control_in_blocks(portable["content"]),
}
def _with_portable_cache_control_in_message(message: object) -> object:
if not isinstance(message, Mapping) or "content" not in message:
return message
content: Final = message["content"]
if isinstance(content, str) or not isinstance(content, Sequence):
return message
return { # mutable-ok: JSON wire format
**message,
"content": [_with_portable_cache_control_in_content_block(block) for block in content],
}
def normalize_cache_control_in_anthropic_payload( # mutable-ok: JSON wire format
payload: Mapping[str, object],
) -> dict[str, object]:
"""
Return a copy of an Anthropic /v1/messages payload with every
``cache_control`` entry reduced to ``{"type": <its type, or "ephemeral">}``,
recursing through message content blocks, system blocks, and tools.
``cache_control`` entry reduced to ``{"type": <its type, or "ephemeral">}``
at the places the Messages API defines it: the request itself, system
blocks, tools, message content blocks, and ``tool_result`` content blocks.
Application data such as ``tool_use.input`` and tool ``input_schema`` is
never touched, even when it happens to contain a ``cache_control`` key.
Anthropic itself accepts prompt-caching extensions such as ``ttl``, but
strict non-Anthropic implementations of the Messages API validate the field
literally and reject the whole request (``cache_control.ttl: 1h is not
supported``, ``cache_control.type is required``), which 400s clients like
Claude Code that always send cache hints. Non-dict ``cache_control`` values
are dropped entirely. The caller's payload is never mutated.
Claude Code that send cache hints. Non-dict ``cache_control`` values are
dropped entirely. The caller's payload is never mutated.
"""
return { # mutable-ok: JSON wire format, as sibling sanitizers
key: _normalized_cache_control(value) if key == "cache_control" else _normalize_cache_control_value(value)
for key, value in payload.items()
if key != "cache_control" or isinstance(value, dict)
portable: Final = _with_portable_cache_control(payload)
scoped: Final = { # mutable-ok: JSON wire format
key: (
_with_portable_cache_control_in_blocks(value)
if key in ("system", "tools")
else [_with_portable_cache_control_in_message(message) for message in value]
if key == "messages" and isinstance(value, Sequence) and not isinstance(value, str)
else value
)
for key, value in portable.items()
}
return scoped
def process_anthropic_headers(headers: httpx.Headers | dict) -> dict:

62
tests/test_litellm/llms/openai_like/messages/test_openai_like_anthropic_messages_transformation.py
View file

@ -438,3 +438,65 @@ def test_json_provider_constraint_opts_into_cache_control_ttl():
assert transform(strict)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral"}
assert transform(lenient)["messages"][0]["content"][0]["cache_control"] == {"type": "ephemeral", "ttl": "1h"}
def test_request_strips_ttl_only_where_the_messages_api_defines_cache_control(config):
"""Regression: the sanitizer must only touch ``cache_control`` where the
Messages API defines it (request, system, tools, content blocks, tool_result
content), never application data such as ``tool_use.input`` or a tool's
``input_schema`` that happens to contain a ``cache_control`` key."""
tool_input = {"cache_control": {"type": "ephemeral", "ttl": "1h"}, "query": "x"}
input_schema = {
"type": "object",
"properties": {"cache_control": {"type": "string", "ttl": "1h"}},
}
messages = [
{
"role": "assistant",
"content": [{"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": tool_input}],
},
{
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "toolu_1",
"cache_control": {"type": "ephemeral", "ttl": "1h"},
"content": [
{"type": "text", "text": "result", "cache_control": {"type": "ephemeral", "ttl": "1h"}}
],
},
{"type": "text", "text": "plain string content stays", "cache_control": {"ttl": "1h"}},
],
},
{"role": "user", "content": "a plain string message"},
]
optional_params = {
"max_tokens": 64,
"cache_control": {"type": "ephemeral", "ttl": "1h"},
"tools": [
{
"name": "lookup",
"input_schema": input_schema,
"cache_control": {"type": "ephemeral", "ttl": "1h"},
}
],
}
payload = config.transform_anthropic_messages_request(
model="some-model",
messages=messages,
anthropic_messages_optional_request_params=optional_params,
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert payload["cache_control"] == {"type": "ephemeral"}
assert payload["tools"][0]["cache_control"] == {"type": "ephemeral"}
assert payload["tools"][0]["input_schema"] == input_schema
assert payload["messages"][0]["content"][0]["input"] == tool_input
tool_result = payload["messages"][1]["content"][0]
assert tool_result["cache_control"] == {"type": "ephemeral"}
assert tool_result["content"][0]["cache_control"] == {"type": "ephemeral"}
assert payload["messages"][1]["content"][1]["cache_control"] == {"type": "ephemeral"}
assert payload["messages"][2] == {"role": "user", "content": "a plain string message"}