mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
fix(sap): preserve cache_control breakpoints on the SAP Orchestration route
Anthropic `cache_control` breakpoints were silently dropped when routing through SAP AI Core Orchestration, so prompt caching never activated and `cache_read_input_tokens` / `cache_creation_input_tokens` stayed 0. Two separate strip points: - `TextContent` had no `cache_control` field, so pydantic discarded it from user-message content blocks. - `SAPMessage` (system/developer) coerced list content to a joined string, dropping both the block structure and the breakpoint. Keep the block form only when a breakpoint is present, so requests without `cache_control` serialize byte-for-byte as before. Assistant and tool messages keep flattening, matching the SAP Cloud SDK for AI, which skips those roles when applying cache_control. Fixes #37866
This commit is contained in:
Joseph Bergin
parent
cd63c7e5a7
commit
2a544e4049
2 changed files with 237 additions and 2 deletions
|
|
@ -24,9 +24,53 @@ def validate_different_content(v: str | dict | list) -> str:
|
|||
raise ValueError("Content must be a string")
|
||||
|
||||
|
||||
def _has_cache_control(v: str | dict | list) -> bool:
|
||||
"""True if any content block carries a ``cache_control`` breakpoint."""
|
||||
if isinstance(v, dict):
|
||||
return v.get("cache_control") is not None
|
||||
if isinstance(v, list):
|
||||
return any(isinstance(item, dict) and item.get("cache_control") is not None for item in v)
|
||||
return False
|
||||
|
||||
|
||||
def validate_cacheable_content(v: str | dict | list) -> str | list:
|
||||
"""Flatten content to a string, keeping the block form when it is cached.
|
||||
|
||||
SAP Orchestration accepts either a plain string or a list of ``text`` blocks for
|
||||
system/developer messages. Flattening unconditionally drops any ``cache_control``
|
||||
breakpoint set on a block, so prompt caching never activates on the ``sap/`` route.
|
||||
Keep the block form only when a breakpoint is present, leaving every other request
|
||||
byte-for-byte unchanged.
|
||||
"""
|
||||
if not _has_cache_control(v):
|
||||
return validate_different_content(v)
|
||||
|
||||
blocks: Final = [v] if isinstance(v, dict) else v
|
||||
kept: Final[list] = []
|
||||
for item in blocks:
|
||||
if isinstance(item, str):
|
||||
if item:
|
||||
kept.append({"type": "text", "text": item})
|
||||
elif isinstance(item, dict) and item.get("text"):
|
||||
kept.append(item)
|
||||
return kept
|
||||
|
||||
|
||||
class CacheControl(BaseModel):
|
||||
"""Prompt-cache breakpoint forwarded to the model provider.
|
||||
|
||||
SAP Orchestration passes this through to Bedrock-hosted Anthropic Claude and
|
||||
Amazon Nova models, which use it to mark where the cached prefix ends.
|
||||
"""
|
||||
|
||||
type_: Literal["ephemeral"] = Field(default="ephemeral", alias="type")
|
||||
ttl: str | None = None
|
||||
|
||||
|
||||
class TextContent(BaseModel):
|
||||
type_: Literal["text"] = Field(default="text", alias="type")
|
||||
text: str
|
||||
cache_control: CacheControl | None = None
|
||||
|
||||
|
||||
class ImageURLContent(BaseModel):
|
||||
|
|
@ -88,9 +132,9 @@ class SAPMessage(BaseModel):
|
|||
"""
|
||||
|
||||
role: Literal["system", "developer"] = "system"
|
||||
content: str
|
||||
content: str | list[TextContent]
|
||||
|
||||
_content_validator = field_validator("content", mode="before")(validate_different_content)
|
||||
_content_validator = field_validator("content", mode="before")(validate_cacheable_content)
|
||||
|
||||
|
||||
class SAPUserMessage(BaseModel):
|
||||
|
|
|
|||
191
tests/test_litellm/llms/sap/chat/test_sap_cache_control.py
Normal file
191
tests/test_litellm/llms/sap/chat/test_sap_cache_control.py
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
"""Regression tests for cache_control passthrough on the SAP Orchestration route.
|
||||
|
||||
SAP Orchestration forwards a ``cache_control`` breakpoint to Bedrock-hosted
|
||||
Anthropic Claude and Amazon Nova models. The transformation used to drop those
|
||||
breakpoints, so prompt caching silently never activated (BerriAI/litellm#37866).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.sap.chat.transformation import (
|
||||
GenAIHubOrchestrationConfig,
|
||||
_messages_to_sap_template,
|
||||
)
|
||||
|
||||
EPHEMERAL = {"type": "ephemeral"}
|
||||
|
||||
|
||||
def _template(result: dict) -> list:
|
||||
return result["config"]["modules"]["prompt_templating"]["prompt"]["template"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_config():
|
||||
config = GenAIHubOrchestrationConfig()
|
||||
config.token_creator = lambda: "Bearer TEST_TOKEN"
|
||||
config._base_url = "https://api.test-sap.com"
|
||||
config._resource_group = "test-group"
|
||||
return config
|
||||
|
||||
|
||||
class TestSAPCacheControl:
|
||||
"""cache_control breakpoints must survive the SAP transformation."""
|
||||
|
||||
def test_system_message_keeps_breakpoint(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "Long prefix", "cache_control": EPHEMERAL}],
|
||||
}
|
||||
]
|
||||
|
||||
template = _messages_to_sap_template(messages)
|
||||
|
||||
assert template == [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "Long prefix", "cache_control": EPHEMERAL}],
|
||||
}
|
||||
]
|
||||
|
||||
def test_user_message_keeps_breakpoint(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Summarize.", "cache_control": EPHEMERAL}],
|
||||
}
|
||||
]
|
||||
|
||||
template = _messages_to_sap_template(messages)
|
||||
|
||||
assert template[0]["content"][0]["cache_control"] == EPHEMERAL
|
||||
|
||||
def test_breakpoint_survives_full_transform_request(self, mock_config):
|
||||
"""The breakpoint must still be there in the body actually sent to SAP."""
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "Cached prefix", "cache_control": EPHEMERAL}],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Question?", "cache_control": EPHEMERAL}],
|
||||
},
|
||||
]
|
||||
|
||||
result = mock_config.transform_request("anthropic--claude-4.5-haiku", messages, {}, {}, {})
|
||||
template = _template(result)
|
||||
|
||||
assert template[0]["content"][0]["cache_control"] == EPHEMERAL
|
||||
assert template[1]["content"][0]["cache_control"] == EPHEMERAL
|
||||
|
||||
def test_ttl_is_preserved(self):
|
||||
"""Anthropic's extended cache TTL must not be dropped either."""
|
||||
cache_control = {"type": "ephemeral", "ttl": "1h"}
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Hi", "cache_control": cache_control}],
|
||||
}
|
||||
]
|
||||
|
||||
template = _messages_to_sap_template(messages)
|
||||
|
||||
assert template[0]["content"][0]["cache_control"] == cache_control
|
||||
|
||||
def test_multiple_breakpoints_are_all_kept(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": "A", "cache_control": EPHEMERAL},
|
||||
{"type": "text", "text": "B"},
|
||||
{"type": "text", "text": "C", "cache_control": EPHEMERAL},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
content = _messages_to_sap_template(messages)[0]["content"]
|
||||
|
||||
assert [block.get("cache_control") for block in content] == [
|
||||
EPHEMERAL,
|
||||
None,
|
||||
EPHEMERAL,
|
||||
]
|
||||
|
||||
def test_plain_string_alongside_cached_block_becomes_a_text_block(self):
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": ["raw string", {"type": "text", "text": "B", "cache_control": EPHEMERAL}],
|
||||
}
|
||||
]
|
||||
|
||||
content = _messages_to_sap_template(messages)[0]["content"]
|
||||
|
||||
assert content == [
|
||||
{"type": "text", "text": "raw string"},
|
||||
{"type": "text", "text": "B", "cache_control": EPHEMERAL},
|
||||
]
|
||||
|
||||
|
||||
class TestSAPCacheControlNoRegression:
|
||||
"""Requests without a breakpoint must be byte-for-byte unchanged."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"message, expected",
|
||||
[
|
||||
(
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
),
|
||||
(
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "A"}, {"type": "text", "text": "B"}],
|
||||
},
|
||||
{"role": "system", "content": "A\nB"},
|
||||
),
|
||||
(
|
||||
{"role": "system", "content": {"type": "text", "text": "solo"}},
|
||||
{"role": "system", "content": "solo"},
|
||||
),
|
||||
(
|
||||
{"role": "system", "content": []},
|
||||
{"role": "system", "content": ""},
|
||||
),
|
||||
(
|
||||
{"role": "developer", "content": [{"type": "text", "text": "D"}]},
|
||||
{"role": "developer", "content": "D"},
|
||||
),
|
||||
(
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "user", "content": "hi"},
|
||||
),
|
||||
(
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi"}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "hi"}]},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_uncached_messages_are_unchanged(self, message, expected):
|
||||
assert _messages_to_sap_template([message]) == [expected]
|
||||
|
||||
def test_assistant_and_tool_messages_still_flatten(self):
|
||||
"""SAP's own SDK skips assistant/tool messages when applying cache_control."""
|
||||
messages = [
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "text", "text": "ans", "cache_control": EPHEMERAL}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [{"type": "text", "text": "res", "cache_control": EPHEMERAL}],
|
||||
},
|
||||
]
|
||||
|
||||
template = _messages_to_sap_template(messages)
|
||||
|
||||
assert template[0]["content"] == "ans"
|
||||
assert template[1]["content"] == "res"
|
||||
Loading…
Add table
Reference in a new issue