From 2a544e4049fc9f4a2d9544917f84adae8af35c44 Mon Sep 17 00:00:00 2001 From: Joseph Bergin Date: Mon, 24 Aug 2026 16:15:15 -0500 Subject: [PATCH] fix(sap): preserve cache_control breakpoints on the SAP Orchestration route Anthropic `cache_control` breakpoints were silently dropped when routing through SAP AI Core Orchestration, so prompt caching never activated and `cache_read_input_tokens` / `cache_creation_input_tokens` stayed 0. Two separate strip points: - `TextContent` had no `cache_control` field, so pydantic discarded it from user-message content blocks. - `SAPMessage` (system/developer) coerced list content to a joined string, dropping both the block structure and the breakpoint. Keep the block form only when a breakpoint is present, so requests without `cache_control` serialize byte-for-byte as before. Assistant and tool messages keep flattening, matching the SAP Cloud SDK for AI, which skips those roles when applying cache_control. Fixes #37866 --- litellm/llms/sap/chat/models.py | 48 ++++- .../llms/sap/chat/test_sap_cache_control.py | 191 ++++++++++++++++++ 2 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/llms/sap/chat/test_sap_cache_control.py diff --git a/litellm/llms/sap/chat/models.py b/litellm/llms/sap/chat/models.py index 5f65c7f715d..791daf18e2d 100644 --- a/litellm/llms/sap/chat/models.py +++ b/litellm/llms/sap/chat/models.py @@ -24,9 +24,53 @@ def validate_different_content(v: str | dict | list) -> str: raise ValueError("Content must be a string") +def _has_cache_control(v: str | dict | list) -> bool: + """True if any content block carries a ``cache_control`` breakpoint.""" + if isinstance(v, dict): + return v.get("cache_control") is not None + if isinstance(v, list): + return any(isinstance(item, dict) and item.get("cache_control") is not None for item in v) + return False + + +def validate_cacheable_content(v: str | dict | list) -> str | list: + """Flatten content to a string, keeping the block form when it is cached. + + SAP Orchestration accepts either a plain string or a list of ``text`` blocks for + system/developer messages. Flattening unconditionally drops any ``cache_control`` + breakpoint set on a block, so prompt caching never activates on the ``sap/`` route. + Keep the block form only when a breakpoint is present, leaving every other request + byte-for-byte unchanged. + """ + if not _has_cache_control(v): + return validate_different_content(v) + + blocks: Final = [v] if isinstance(v, dict) else v + kept: Final[list] = [] + for item in blocks: + if isinstance(item, str): + if item: + kept.append({"type": "text", "text": item}) + elif isinstance(item, dict) and item.get("text"): + kept.append(item) + return kept + + +class CacheControl(BaseModel): + """Prompt-cache breakpoint forwarded to the model provider. + + SAP Orchestration passes this through to Bedrock-hosted Anthropic Claude and + Amazon Nova models, which use it to mark where the cached prefix ends. + """ + + type_: Literal["ephemeral"] = Field(default="ephemeral", alias="type") + ttl: str | None = None + + class TextContent(BaseModel): type_: Literal["text"] = Field(default="text", alias="type") text: str + cache_control: CacheControl | None = None class ImageURLContent(BaseModel): @@ -88,9 +132,9 @@ class SAPMessage(BaseModel): """ role: Literal["system", "developer"] = "system" - content: str + content: str | list[TextContent] - _content_validator = field_validator("content", mode="before")(validate_different_content) + _content_validator = field_validator("content", mode="before")(validate_cacheable_content) class SAPUserMessage(BaseModel): diff --git a/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py b/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py new file mode 100644 index 00000000000..75a2890e7e6 --- /dev/null +++ b/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py @@ -0,0 +1,191 @@ +"""Regression tests for cache_control passthrough on the SAP Orchestration route. + +SAP Orchestration forwards a ``cache_control`` breakpoint to Bedrock-hosted +Anthropic Claude and Amazon Nova models. The transformation used to drop those +breakpoints, so prompt caching silently never activated (BerriAI/litellm#37866). +""" + +import pytest + +from litellm.llms.sap.chat.transformation import ( + GenAIHubOrchestrationConfig, + _messages_to_sap_template, +) + +EPHEMERAL = {"type": "ephemeral"} + + +def _template(result: dict) -> list: + return result["config"]["modules"]["prompt_templating"]["prompt"]["template"] + + +@pytest.fixture +def mock_config(): + config = GenAIHubOrchestrationConfig() + config.token_creator = lambda: "Bearer TEST_TOKEN" + config._base_url = "https://api.test-sap.com" + config._resource_group = "test-group" + return config + + +class TestSAPCacheControl: + """cache_control breakpoints must survive the SAP transformation.""" + + def test_system_message_keeps_breakpoint(self): + messages = [ + { + "role": "system", + "content": [{"type": "text", "text": "Long prefix", "cache_control": EPHEMERAL}], + } + ] + + template = _messages_to_sap_template(messages) + + assert template == [ + { + "role": "system", + "content": [{"type": "text", "text": "Long prefix", "cache_control": EPHEMERAL}], + } + ] + + def test_user_message_keeps_breakpoint(self): + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "Summarize.", "cache_control": EPHEMERAL}], + } + ] + + template = _messages_to_sap_template(messages) + + assert template[0]["content"][0]["cache_control"] == EPHEMERAL + + def test_breakpoint_survives_full_transform_request(self, mock_config): + """The breakpoint must still be there in the body actually sent to SAP.""" + messages = [ + { + "role": "system", + "content": [{"type": "text", "text": "Cached prefix", "cache_control": EPHEMERAL}], + }, + { + "role": "user", + "content": [{"type": "text", "text": "Question?", "cache_control": EPHEMERAL}], + }, + ] + + result = mock_config.transform_request("anthropic--claude-4.5-haiku", messages, {}, {}, {}) + template = _template(result) + + assert template[0]["content"][0]["cache_control"] == EPHEMERAL + assert template[1]["content"][0]["cache_control"] == EPHEMERAL + + def test_ttl_is_preserved(self): + """Anthropic's extended cache TTL must not be dropped either.""" + cache_control = {"type": "ephemeral", "ttl": "1h"} + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "Hi", "cache_control": cache_control}], + } + ] + + template = _messages_to_sap_template(messages) + + assert template[0]["content"][0]["cache_control"] == cache_control + + def test_multiple_breakpoints_are_all_kept(self): + messages = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A", "cache_control": EPHEMERAL}, + {"type": "text", "text": "B"}, + {"type": "text", "text": "C", "cache_control": EPHEMERAL}, + ], + } + ] + + content = _messages_to_sap_template(messages)[0]["content"] + + assert [block.get("cache_control") for block in content] == [ + EPHEMERAL, + None, + EPHEMERAL, + ] + + def test_plain_string_alongside_cached_block_becomes_a_text_block(self): + messages = [ + { + "role": "system", + "content": ["raw string", {"type": "text", "text": "B", "cache_control": EPHEMERAL}], + } + ] + + content = _messages_to_sap_template(messages)[0]["content"] + + assert content == [ + {"type": "text", "text": "raw string"}, + {"type": "text", "text": "B", "cache_control": EPHEMERAL}, + ] + + +class TestSAPCacheControlNoRegression: + """Requests without a breakpoint must be byte-for-byte unchanged.""" + + @pytest.mark.parametrize( + "message, expected", + [ + ( + {"role": "system", "content": "You are helpful."}, + {"role": "system", "content": "You are helpful."}, + ), + ( + { + "role": "system", + "content": [{"type": "text", "text": "A"}, {"type": "text", "text": "B"}], + }, + {"role": "system", "content": "A\nB"}, + ), + ( + {"role": "system", "content": {"type": "text", "text": "solo"}}, + {"role": "system", "content": "solo"}, + ), + ( + {"role": "system", "content": []}, + {"role": "system", "content": ""}, + ), + ( + {"role": "developer", "content": [{"type": "text", "text": "D"}]}, + {"role": "developer", "content": "D"}, + ), + ( + {"role": "user", "content": "hi"}, + {"role": "user", "content": "hi"}, + ), + ( + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + ), + ], + ) + def test_uncached_messages_are_unchanged(self, message, expected): + assert _messages_to_sap_template([message]) == [expected] + + def test_assistant_and_tool_messages_still_flatten(self): + """SAP's own SDK skips assistant/tool messages when applying cache_control.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "text", "text": "ans", "cache_control": EPHEMERAL}], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "res", "cache_control": EPHEMERAL}], + }, + ] + + template = _messages_to_sap_template(messages) + + assert template[0]["content"] == "ans" + assert template[1]["content"] == "res"