From 5af191691417fdf17ab859acfd5e56c647909de1 Mon Sep 17 00:00:00 2001 From: Joseph Bergin Date: Mon, 24 Aug 2026 16:15:15 -0500 Subject: [PATCH 1/3] fix(sap): preserve cache_control breakpoints on the SAP Orchestration route Anthropic `cache_control` breakpoints were silently dropped when routing through SAP AI Core Orchestration, so prompt caching never activated and `cache_read_input_tokens` / `cache_creation_input_tokens` stayed 0. Two separate strip points: - `TextContent` had no `cache_control` field, so pydantic discarded it from user-message content blocks. - `SAPMessage` (system/developer) coerced list content to a joined string, dropping both the block structure and the breakpoint. Keep the block form only when a breakpoint is present, so requests without `cache_control` serialize byte-for-byte as before. Assistant and tool messages keep flattening, matching the SAP Cloud SDK for AI, which skips those roles when applying cache_control. Fixes #37866 --- litellm/llms/sap/chat/models.py | 48 ++++- .../llms/sap/chat/test_sap_cache_control.py | 191 ++++++++++++++++++ 2 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/llms/sap/chat/test_sap_cache_control.py diff --git a/litellm/llms/sap/chat/models.py b/litellm/llms/sap/chat/models.py index 5f65c7f715d..791daf18e2d 100644 --- a/litellm/llms/sap/chat/models.py +++ b/litellm/llms/sap/chat/models.py @@ -24,9 +24,53 @@ def validate_different_content(v: str | dict | list) -> str: raise ValueError("Content must be a string") +def _has_cache_control(v: str | dict | list) -> bool: + """True if any content block carries a ``cache_control`` breakpoint.""" + if isinstance(v, dict): + return v.get("cache_control") is not None + if isinstance(v, list): + return any(isinstance(item, dict) and item.get("cache_control") is not None for item in v) + return False + + +def validate_cacheable_content(v: str | dict | list) -> str | list: + """Flatten content to a string, keeping the block form when it is cached. + + SAP Orchestration accepts either a plain string or a list of ``text`` blocks for + system/developer messages. Flattening unconditionally drops any ``cache_control`` + breakpoint set on a block, so prompt caching never activates on the ``sap/`` route. + Keep the block form only when a breakpoint is present, leaving every other request + byte-for-byte unchanged. + """ + if not _has_cache_control(v): + return validate_different_content(v) + + blocks: Final = [v] if isinstance(v, dict) else v + kept: Final[list] = [] + for item in blocks: + if isinstance(item, str): + if item: + kept.append({"type": "text", "text": item}) + elif isinstance(item, dict) and item.get("text"): + kept.append(item) + return kept + + +class CacheControl(BaseModel): + """Prompt-cache breakpoint forwarded to the model provider. + + SAP Orchestration passes this through to Bedrock-hosted Anthropic Claude and + Amazon Nova models, which use it to mark where the cached prefix ends. + """ + + type_: Literal["ephemeral"] = Field(default="ephemeral", alias="type") + ttl: str | None = None + + class TextContent(BaseModel): type_: Literal["text"] = Field(default="text", alias="type") text: str + cache_control: CacheControl | None = None class ImageURLContent(BaseModel): @@ -88,9 +132,9 @@ class SAPMessage(BaseModel): """ role: Literal["system", "developer"] = "system" - content: str + content: str | list[TextContent] - _content_validator = field_validator("content", mode="before")(validate_different_content) + _content_validator = field_validator("content", mode="before")(validate_cacheable_content) class SAPUserMessage(BaseModel): diff --git a/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py b/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py new file mode 100644 index 00000000000..75a2890e7e6 --- /dev/null +++ b/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py @@ -0,0 +1,191 @@ +"""Regression tests for cache_control passthrough on the SAP Orchestration route. + +SAP Orchestration forwards a ``cache_control`` breakpoint to Bedrock-hosted +Anthropic Claude and Amazon Nova models. The transformation used to drop those +breakpoints, so prompt caching silently never activated (BerriAI/litellm#37866). +""" + +import pytest + +from litellm.llms.sap.chat.transformation import ( + GenAIHubOrchestrationConfig, + _messages_to_sap_template, +) + +EPHEMERAL = {"type": "ephemeral"} + + +def _template(result: dict) -> list: + return result["config"]["modules"]["prompt_templating"]["prompt"]["template"] + + +@pytest.fixture +def mock_config(): + config = GenAIHubOrchestrationConfig() + config.token_creator = lambda: "Bearer TEST_TOKEN" + config._base_url = "https://api.test-sap.com" + config._resource_group = "test-group" + return config + + +class TestSAPCacheControl: + """cache_control breakpoints must survive the SAP transformation.""" + + def test_system_message_keeps_breakpoint(self): + messages = [ + { + "role": "system", + "content": [{"type": "text", "text": "Long prefix", "cache_control": EPHEMERAL}], + } + ] + + template = _messages_to_sap_template(messages) + + assert template == [ + { + "role": "system", + "content": [{"type": "text", "text": "Long prefix", "cache_control": EPHEMERAL}], + } + ] + + def test_user_message_keeps_breakpoint(self): + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "Summarize.", "cache_control": EPHEMERAL}], + } + ] + + template = _messages_to_sap_template(messages) + + assert template[0]["content"][0]["cache_control"] == EPHEMERAL + + def test_breakpoint_survives_full_transform_request(self, mock_config): + """The breakpoint must still be there in the body actually sent to SAP.""" + messages = [ + { + "role": "system", + "content": [{"type": "text", "text": "Cached prefix", "cache_control": EPHEMERAL}], + }, + { + "role": "user", + "content": [{"type": "text", "text": "Question?", "cache_control": EPHEMERAL}], + }, + ] + + result = mock_config.transform_request("anthropic--claude-4.5-haiku", messages, {}, {}, {}) + template = _template(result) + + assert template[0]["content"][0]["cache_control"] == EPHEMERAL + assert template[1]["content"][0]["cache_control"] == EPHEMERAL + + def test_ttl_is_preserved(self): + """Anthropic's extended cache TTL must not be dropped either.""" + cache_control = {"type": "ephemeral", "ttl": "1h"} + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "Hi", "cache_control": cache_control}], + } + ] + + template = _messages_to_sap_template(messages) + + assert template[0]["content"][0]["cache_control"] == cache_control + + def test_multiple_breakpoints_are_all_kept(self): + messages = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A", "cache_control": EPHEMERAL}, + {"type": "text", "text": "B"}, + {"type": "text", "text": "C", "cache_control": EPHEMERAL}, + ], + } + ] + + content = _messages_to_sap_template(messages)[0]["content"] + + assert [block.get("cache_control") for block in content] == [ + EPHEMERAL, + None, + EPHEMERAL, + ] + + def test_plain_string_alongside_cached_block_becomes_a_text_block(self): + messages = [ + { + "role": "system", + "content": ["raw string", {"type": "text", "text": "B", "cache_control": EPHEMERAL}], + } + ] + + content = _messages_to_sap_template(messages)[0]["content"] + + assert content == [ + {"type": "text", "text": "raw string"}, + {"type": "text", "text": "B", "cache_control": EPHEMERAL}, + ] + + +class TestSAPCacheControlNoRegression: + """Requests without a breakpoint must be byte-for-byte unchanged.""" + + @pytest.mark.parametrize( + "message, expected", + [ + ( + {"role": "system", "content": "You are helpful."}, + {"role": "system", "content": "You are helpful."}, + ), + ( + { + "role": "system", + "content": [{"type": "text", "text": "A"}, {"type": "text", "text": "B"}], + }, + {"role": "system", "content": "A\nB"}, + ), + ( + {"role": "system", "content": {"type": "text", "text": "solo"}}, + {"role": "system", "content": "solo"}, + ), + ( + {"role": "system", "content": []}, + {"role": "system", "content": ""}, + ), + ( + {"role": "developer", "content": [{"type": "text", "text": "D"}]}, + {"role": "developer", "content": "D"}, + ), + ( + {"role": "user", "content": "hi"}, + {"role": "user", "content": "hi"}, + ), + ( + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + ), + ], + ) + def test_uncached_messages_are_unchanged(self, message, expected): + assert _messages_to_sap_template([message]) == [expected] + + def test_assistant_and_tool_messages_still_flatten(self): + """SAP's own SDK skips assistant/tool messages when applying cache_control.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "text", "text": "ans", "cache_control": EPHEMERAL}], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "res", "cache_control": EPHEMERAL}], + }, + ] + + template = _messages_to_sap_template(messages) + + assert template[0]["content"] == "ans" + assert template[1]["content"] == "res" From 9f9fbe4991939291a3e4856daf94904f9ebc4769 Mon Sep 17 00:00:00 2001 From: Joseph Bergin Date: Mon, 24 Aug 2026 17:11:19 -0500 Subject: [PATCH 2/3] fix(sap): annotate cache_control helpers with read-only collection views The type-discipline gate (LIT001) flags mutable `dict`/`list` in annotations. Use Mapping/Sequence views and build the normalized block list functionally instead of appending in a loop, bringing LIT001 back to the base count. --- litellm/llms/sap/chat/models.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/litellm/llms/sap/chat/models.py b/litellm/llms/sap/chat/models.py index 791daf18e2d..f6786626a8f 100644 --- a/litellm/llms/sap/chat/models.py +++ b/litellm/llms/sap/chat/models.py @@ -1,4 +1,5 @@ import warnings +from collections.abc import Mapping, Sequence from enum import Enum from typing import Final, Literal @@ -24,16 +25,18 @@ def validate_different_content(v: str | dict | list) -> str: raise ValueError("Content must be a string") -def _has_cache_control(v: str | dict | list) -> bool: +def _has_cache_control(v: str | Mapping[str, object] | Sequence[object]) -> bool: """True if any content block carries a ``cache_control`` breakpoint.""" - if isinstance(v, dict): + if isinstance(v, Mapping): return v.get("cache_control") is not None - if isinstance(v, list): - return any(isinstance(item, dict) and item.get("cache_control") is not None for item in v) - return False + if isinstance(v, str): + return False + return any(isinstance(item, Mapping) and item.get("cache_control") is not None for item in v) -def validate_cacheable_content(v: str | dict | list) -> str | list: +def validate_cacheable_content( + v: str | Mapping[str, object] | Sequence[object], +) -> str | Sequence[Mapping[str, object]]: """Flatten content to a string, keeping the block form when it is cached. SAP Orchestration accepts either a plain string or a list of ``text`` blocks for @@ -45,15 +48,9 @@ def validate_cacheable_content(v: str | dict | list) -> str | list: if not _has_cache_control(v): return validate_different_content(v) - blocks: Final = [v] if isinstance(v, dict) else v - kept: Final[list] = [] - for item in blocks: - if isinstance(item, str): - if item: - kept.append({"type": "text", "text": item}) - elif isinstance(item, dict) and item.get("text"): - kept.append(item) - return kept + blocks: Final = (v,) if isinstance(v, Mapping) else v + normalized: Final = ({"type": "text", "text": item} if isinstance(item, str) else item for item in blocks) + return [block for block in normalized if isinstance(block, Mapping) and block.get("text")] class CacheControl(BaseModel): @@ -132,7 +129,7 @@ class SAPMessage(BaseModel): """ role: Literal["system", "developer"] = "system" - content: str | list[TextContent] + content: str | Sequence[TextContent] _content_validator = field_validator("content", mode="before")(validate_cacheable_content) From 10752af8221af8806758c1997edba87f59abb90c Mon Sep 17 00:00:00 2001 From: Joseph Bergin Date: Wed, 26 Aug 2026 15:45:17 -0500 Subject: [PATCH 3/3] test(sap): fold cache_control coverage into the mapped transformation file tests/test_litellm/ mirrors litellm/ one to one, and the convention for a bug fix is to extend the existing mapped test file rather than add a sibling next to it, so the cache_control regressions move into test_sap_transformation.py and test_sap_cache_control.py goes away The helpers also drop the docstrings they were carrying, since comments are reserved here for genuinely complex business logic, and validate_cacheable_content now returns a tuple instead of a list to stay on the immutable side of LIT001 --- litellm/llms/sap/chat/models.py | 21 +- .../llms/sap/chat/test_sap_cache_control.py | 191 --------------- .../llms/sap/chat/test_sap_transformation.py | 221 ++++++++++-------- 3 files changed, 127 insertions(+), 306 deletions(-) delete mode 100644 tests/test_litellm/llms/sap/chat/test_sap_cache_control.py diff --git a/litellm/llms/sap/chat/models.py b/litellm/llms/sap/chat/models.py index f6786626a8f..42a9f9161c8 100644 --- a/litellm/llms/sap/chat/models.py +++ b/litellm/llms/sap/chat/models.py @@ -6,7 +6,7 @@ from typing import Final, Literal from pydantic import BaseModel, Field, field_validator, model_validator -def validate_different_content(v: str | dict | list) -> str: +def validate_different_content(v: str | Mapping[str, object] | Sequence[object]) -> str: if v in ((), {}, []): return "" elif isinstance(v, dict) and "text" in v: @@ -26,7 +26,6 @@ def validate_different_content(v: str | dict | list) -> str: def _has_cache_control(v: str | Mapping[str, object] | Sequence[object]) -> bool: - """True if any content block carries a ``cache_control`` breakpoint.""" if isinstance(v, Mapping): return v.get("cache_control") is not None if isinstance(v, str): @@ -36,30 +35,16 @@ def _has_cache_control(v: str | Mapping[str, object] | Sequence[object]) -> bool def validate_cacheable_content( v: str | Mapping[str, object] | Sequence[object], -) -> str | Sequence[Mapping[str, object]]: - """Flatten content to a string, keeping the block form when it is cached. - - SAP Orchestration accepts either a plain string or a list of ``text`` blocks for - system/developer messages. Flattening unconditionally drops any ``cache_control`` - breakpoint set on a block, so prompt caching never activates on the ``sap/`` route. - Keep the block form only when a breakpoint is present, leaving every other request - byte-for-byte unchanged. - """ +) -> str | tuple[Mapping[str, object], ...]: if not _has_cache_control(v): return validate_different_content(v) blocks: Final = (v,) if isinstance(v, Mapping) else v normalized: Final = ({"type": "text", "text": item} if isinstance(item, str) else item for item in blocks) - return [block for block in normalized if isinstance(block, Mapping) and block.get("text")] + return tuple(block for block in normalized if isinstance(block, Mapping) and block.get("text")) class CacheControl(BaseModel): - """Prompt-cache breakpoint forwarded to the model provider. - - SAP Orchestration passes this through to Bedrock-hosted Anthropic Claude and - Amazon Nova models, which use it to mark where the cached prefix ends. - """ - type_: Literal["ephemeral"] = Field(default="ephemeral", alias="type") ttl: str | None = None diff --git a/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py b/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py deleted file mode 100644 index 75a2890e7e6..00000000000 --- a/tests/test_litellm/llms/sap/chat/test_sap_cache_control.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Regression tests for cache_control passthrough on the SAP Orchestration route. - -SAP Orchestration forwards a ``cache_control`` breakpoint to Bedrock-hosted -Anthropic Claude and Amazon Nova models. The transformation used to drop those -breakpoints, so prompt caching silently never activated (BerriAI/litellm#37866). -""" - -import pytest - -from litellm.llms.sap.chat.transformation import ( - GenAIHubOrchestrationConfig, - _messages_to_sap_template, -) - -EPHEMERAL = {"type": "ephemeral"} - - -def _template(result: dict) -> list: - return result["config"]["modules"]["prompt_templating"]["prompt"]["template"] - - -@pytest.fixture -def mock_config(): - config = GenAIHubOrchestrationConfig() - config.token_creator = lambda: "Bearer TEST_TOKEN" - config._base_url = "https://api.test-sap.com" - config._resource_group = "test-group" - return config - - -class TestSAPCacheControl: - """cache_control breakpoints must survive the SAP transformation.""" - - def test_system_message_keeps_breakpoint(self): - messages = [ - { - "role": "system", - "content": [{"type": "text", "text": "Long prefix", "cache_control": EPHEMERAL}], - } - ] - - template = _messages_to_sap_template(messages) - - assert template == [ - { - "role": "system", - "content": [{"type": "text", "text": "Long prefix", "cache_control": EPHEMERAL}], - } - ] - - def test_user_message_keeps_breakpoint(self): - messages = [ - { - "role": "user", - "content": [{"type": "text", "text": "Summarize.", "cache_control": EPHEMERAL}], - } - ] - - template = _messages_to_sap_template(messages) - - assert template[0]["content"][0]["cache_control"] == EPHEMERAL - - def test_breakpoint_survives_full_transform_request(self, mock_config): - """The breakpoint must still be there in the body actually sent to SAP.""" - messages = [ - { - "role": "system", - "content": [{"type": "text", "text": "Cached prefix", "cache_control": EPHEMERAL}], - }, - { - "role": "user", - "content": [{"type": "text", "text": "Question?", "cache_control": EPHEMERAL}], - }, - ] - - result = mock_config.transform_request("anthropic--claude-4.5-haiku", messages, {}, {}, {}) - template = _template(result) - - assert template[0]["content"][0]["cache_control"] == EPHEMERAL - assert template[1]["content"][0]["cache_control"] == EPHEMERAL - - def test_ttl_is_preserved(self): - """Anthropic's extended cache TTL must not be dropped either.""" - cache_control = {"type": "ephemeral", "ttl": "1h"} - messages = [ - { - "role": "user", - "content": [{"type": "text", "text": "Hi", "cache_control": cache_control}], - } - ] - - template = _messages_to_sap_template(messages) - - assert template[0]["content"][0]["cache_control"] == cache_control - - def test_multiple_breakpoints_are_all_kept(self): - messages = [ - { - "role": "system", - "content": [ - {"type": "text", "text": "A", "cache_control": EPHEMERAL}, - {"type": "text", "text": "B"}, - {"type": "text", "text": "C", "cache_control": EPHEMERAL}, - ], - } - ] - - content = _messages_to_sap_template(messages)[0]["content"] - - assert [block.get("cache_control") for block in content] == [ - EPHEMERAL, - None, - EPHEMERAL, - ] - - def test_plain_string_alongside_cached_block_becomes_a_text_block(self): - messages = [ - { - "role": "system", - "content": ["raw string", {"type": "text", "text": "B", "cache_control": EPHEMERAL}], - } - ] - - content = _messages_to_sap_template(messages)[0]["content"] - - assert content == [ - {"type": "text", "text": "raw string"}, - {"type": "text", "text": "B", "cache_control": EPHEMERAL}, - ] - - -class TestSAPCacheControlNoRegression: - """Requests without a breakpoint must be byte-for-byte unchanged.""" - - @pytest.mark.parametrize( - "message, expected", - [ - ( - {"role": "system", "content": "You are helpful."}, - {"role": "system", "content": "You are helpful."}, - ), - ( - { - "role": "system", - "content": [{"type": "text", "text": "A"}, {"type": "text", "text": "B"}], - }, - {"role": "system", "content": "A\nB"}, - ), - ( - {"role": "system", "content": {"type": "text", "text": "solo"}}, - {"role": "system", "content": "solo"}, - ), - ( - {"role": "system", "content": []}, - {"role": "system", "content": ""}, - ), - ( - {"role": "developer", "content": [{"type": "text", "text": "D"}]}, - {"role": "developer", "content": "D"}, - ), - ( - {"role": "user", "content": "hi"}, - {"role": "user", "content": "hi"}, - ), - ( - {"role": "user", "content": [{"type": "text", "text": "hi"}]}, - {"role": "user", "content": [{"type": "text", "text": "hi"}]}, - ), - ], - ) - def test_uncached_messages_are_unchanged(self, message, expected): - assert _messages_to_sap_template([message]) == [expected] - - def test_assistant_and_tool_messages_still_flatten(self): - """SAP's own SDK skips assistant/tool messages when applying cache_control.""" - messages = [ - { - "role": "assistant", - "content": [{"type": "text", "text": "ans", "cache_control": EPHEMERAL}], - }, - { - "role": "tool", - "tool_call_id": "call_1", - "content": [{"type": "text", "text": "res", "cache_control": EPHEMERAL}], - }, - ] - - template = _messages_to_sap_template(messages) - - assert template[0]["content"] == "ans" - assert template[1]["content"] == "res" diff --git a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py index 3601bdd0d5e..110e0387c6d 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py @@ -34,9 +34,7 @@ class TestSAPTransformationIntegration: result = mock_config.transform_request(model, messages, optional_params, {}, {}) - model_params = result["config"]["modules"]["prompt_templating"]["model"][ - "params" - ] + model_params = result["config"]["modules"]["prompt_templating"]["model"]["params"] assert "temperature" in model_params assert "frequency_penalty" in model_params @@ -44,18 +42,16 @@ class TestSAPTransformationIntegration: assert "model_version" not in model_params assert "tools" not in model_params - model_version = result["config"]["modules"]["prompt_templating"]["model"][ - "version" - ] + model_version = result["config"]["modules"]["prompt_templating"]["model"]["version"] assert model_version == "v1.5" prompt = result["config"]["modules"]["prompt_templating"]["prompt"] if "tools" in prompt: assert isinstance(prompt["tools"], list) for tool in prompt["tools"]: - assert ( - tool["function"]["parameters"]["type"] == "object" - ), "SAP API requires parameters.type == 'object'" + assert tool["function"]["parameters"]["type"] == "object", ( + "SAP API requires parameters.type == 'object'" + ) assert "properties" in tool["function"]["parameters"] def test_transform_request_parameter_handling_robustness(self, mock_config): @@ -96,33 +92,25 @@ class TestSAPTransformationIntegration: for i, test_case in enumerate(test_cases): filtered_params = { - k: v - for k, v in test_case["params"].items() - if k not in {"tools", "model_version", "deployment_url"} + k: v for k, v in test_case["params"].items() if k not in {"tools", "model_version", "deployment_url"} } for expected_param in test_case["expected_in_model"]: - assert ( - expected_param in filtered_params - ), f"Case {i + 1}: {expected_param} should be in model params" + assert expected_param in filtered_params, f"Case {i + 1}: {expected_param} should be in model params" for excluded_param in test_case["expected_excluded"]: - assert ( - excluded_param not in filtered_params - ), f"Case {i + 1}: {excluded_param} should be excluded from model params" + assert excluded_param not in filtered_params, ( + f"Case {i + 1}: {excluded_param} should be excluded from model params" + ) - result = mock_config.transform_request( - model, messages, test_case["params"], {}, {} - ) + result = mock_config.transform_request(model, messages, test_case["params"], {}, {}) if result and "config" in result: - model_params = result["config"]["modules"]["prompt_templating"][ - "model" - ]["params"] + model_params = result["config"]["modules"]["prompt_templating"]["model"]["params"] for excluded_param in test_case["expected_excluded"]: - assert ( - excluded_param not in model_params - ), f"Case {i + 1}: {excluded_param} should not be in actual model params" + assert excluded_param not in model_params, ( + f"Case {i + 1}: {excluded_param} should not be in actual model params" + ) def test_config_transform_with_response_format_json_object(self, mock_config): expected_dict = { @@ -145,9 +133,7 @@ class TestSAPTransformationIntegration: } config = mock_config.transform_request( model="gpt-4o", - messages=[ - {"role": "user", "content": "First man on the moon, answer in json"} - ], + messages=[{"role": "user", "content": "First man on the moon, answer in json"}], optional_params={ "response_format": {"type": "json_object"}, "deployment_url": "shouldn't be in results", @@ -189,9 +175,7 @@ class TestSAPTransformationIntegration: config = mock_config.transform_request( model="gpt-4o", - messages=[ - {"role": "user", "content": "First man on the moon, answer in json"} - ], + messages=[{"role": "user", "content": "First man on the moon, answer in json"}], optional_params={ "response_format": expected_response_format, "deployment_url": "shouldn't be in results", @@ -199,27 +183,15 @@ class TestSAPTransformationIntegration: litellm_params={}, headers={}, ) - assert ( - config["config"]["modules"]["prompt_templating"]["prompt"][ - "response_format" - ] - == expected_response_format - ) - assert ( - len(config["config"]["modules"]["prompt_templating"]["model"]["params"]) - == 0 - ) + assert config["config"]["modules"]["prompt_templating"]["prompt"]["response_format"] == expected_response_format + assert len(config["config"]["modules"]["prompt_templating"]["model"]["params"]) == 0 def test_config_transform_with_stream(self, mock_config): expected_dict = { "config": { "modules": { "prompt_templating": { - "prompt": { - "template": [ - {"role": "user", "content": "Hello, how are you?"} - ] - }, + "prompt": {"template": [{"role": "user", "content": "Hello, how are you?"}]}, "model": { "name": "anthropic--claude-4-sonnet", "params": {}, @@ -257,9 +229,7 @@ class TestSAPTransformationIntegration: headers={}, ) - assert config["config"]["modules"]["prompt_templating"]["prompt"][ - "defaults" - ] == {"user_query": "default value"} + assert config["config"]["modules"]["prompt_templating"]["prompt"]["defaults"] == {"user_query": "default value"} assert config["config"]["modules"]["prompt_templating"]["model"]["params"] == {} def test_sap_placeholder_values(self, mock_config): @@ -323,14 +293,8 @@ class TestSAPTransformationIntegration: assert config["placeholder_values"] == placeholder_values modules = config["config"]["modules"] assert modules["grounding"]["type"] == "document_grounding_service" - assert ( - modules["grounding"]["config"]["placeholders"]["output"] - == "grounding_response" - ) - assert ( - modules["grounding"]["config"]["filters"][0]["data_repository_type"] - == "vector" - ) + assert modules["grounding"]["config"]["placeholders"]["output"] == "grounding_response" + assert modules["grounding"]["config"]["filters"][0]["data_repository_type"] == "vector" assert modules["prompt_templating"]["model"]["params"] == {} def test_grounding_search_config_rejects_both_count_fields(self, mock_config): @@ -442,10 +406,7 @@ class TestSAPTransformationIntegration: headers={}, ) - assert ( - "For using SAP Filtering Module you must provide at least one property" - in str(exc_info.value) - ) + assert "For using SAP Filtering Module you must provide at least one property" in str(exc_info.value) def test_sap_masking(self, mock_config): masking_config = { @@ -509,9 +470,7 @@ class TestSAPTransformationIntegration: headers={}, ) - assert "must set exactly one of: 'providers' or 'masking_providers'" in str( - exc_info.value - ) + assert "must set exactly one of: 'providers' or 'masking_providers'" in str(exc_info.value) def test_masking_providers_deprecated_emits_warning(self, mock_config): masking_config = { @@ -533,8 +492,7 @@ class TestSAPTransformationIntegration: headers={}, ) assert any( - issubclass(warning.category, DeprecationWarning) - and "masking_providers" in str(warning.message) + issubclass(warning.category, DeprecationWarning) and "masking_providers" in str(warning.message) for warning in w ), "Expected DeprecationWarning for 'masking_providers'" @@ -573,10 +531,7 @@ class TestSAPTransformationIntegration: headers={}, ) - assert ( - "TranslationModuleConfig requires at least one of 'input' or 'output'" - in str(exc_info.value) - ) + assert "TranslationModuleConfig requires at least one of 'input' or 'output'" in str(exc_info.value) def test_sap_multiple_modules(self, mock_config): translation_config = { @@ -611,31 +566,103 @@ class TestSAPTransformationIntegration: assert translation["input"]["config"]["source_language"] == "en-US" assert translation["input"]["config"]["target_language"] == "de-DE" assert translation["output"]["config"]["target_language"] == "fr-FR" + assert config["config"]["modules"][1]["prompt_templating"]["model"]["name"] == "gpt-5" + assert config["config"]["modules"][0]["prompt_templating"]["model"]["name"] == "gpt-4o" + assert config["config"]["modules"][0]["prompt_templating"]["model"]["params"] == {} assert ( - config["config"]["modules"][1]["prompt_templating"]["model"]["name"] - == "gpt-5" - ) - assert ( - config["config"]["modules"][0]["prompt_templating"]["model"]["name"] - == "gpt-4o" - ) - assert ( - config["config"]["modules"][0]["prompt_templating"]["model"]["params"] - == {} - ) - assert ( - config["config"]["modules"][1]["prompt_templating"]["prompt"][ - "template" - ][0]["content"] + config["config"]["modules"][1]["prompt_templating"]["prompt"]["template"][0]["content"] == "Hello world!" ) - assert ( - config["config"]["modules"][0]["prompt_templating"]["prompt"][ - "template" - ][0]["content"] - == "Hello." - ) - assert ( - config["config"]["modules"][1]["translation"]["input"]["type"] - == "sap_document_translation" - ) + assert config["config"]["modules"][0]["prompt_templating"]["prompt"]["template"][0]["content"] == "Hello." + assert config["config"]["modules"][1]["translation"]["input"]["type"] == "sap_document_translation" + + @pytest.mark.parametrize( + "message, expected", + [ + ({"role": "system", "content": "You are helpful."}, {"role": "system", "content": "You are helpful."}), + ( + {"role": "system", "content": [{"type": "text", "text": "A"}, {"type": "text", "text": "B"}]}, + {"role": "system", "content": "A\nB"}, + ), + ({"role": "system", "content": {"type": "text", "text": "solo"}}, {"role": "system", "content": "solo"}), + ({"role": "system", "content": []}, {"role": "system", "content": ""}), + ({"role": "developer", "content": [{"type": "text", "text": "D"}]}, {"role": "developer", "content": "D"}), + ({"role": "user", "content": "hi"}, {"role": "user", "content": "hi"}), + ( + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + ), + ], + ) + def test_messages_without_cache_control_are_unchanged(self, message, expected): + """Messages with no breakpoint keep the exact shape they had before cache_control support.""" + from litellm.llms.sap.chat.transformation import _messages_to_sap_template + + assert _messages_to_sap_template([message]) == [expected] + + def test_cache_control_survives_transform_request(self, mock_config): + """Breakpoints on system and user blocks reach the body sent to SAP.""" + ephemeral = {"type": "ephemeral"} + messages = [ + {"role": "system", "content": [{"type": "text", "text": "Cached prefix", "cache_control": ephemeral}]}, + {"role": "user", "content": [{"type": "text", "text": "Question?", "cache_control": ephemeral}]}, + ] + + result = mock_config.transform_request("anthropic--claude-4.5-haiku", messages, {}, {}, {}) + + template = result["config"]["modules"]["prompt_templating"]["prompt"]["template"] + assert template[0]["content"][0]["cache_control"] == ephemeral + assert template[1]["content"][0]["cache_control"] == ephemeral + + def test_cache_control_ttl_is_preserved(self): + """Anthropic's extended cache TTL rides along with the breakpoint.""" + from litellm.llms.sap.chat.transformation import _messages_to_sap_template + + cache_control = {"type": "ephemeral", "ttl": "1h"} + messages = [{"role": "user", "content": [{"type": "text", "text": "Hi", "cache_control": cache_control}]}] + + template = _messages_to_sap_template(messages) + + assert template[0]["content"][0]["cache_control"] == cache_control + + def test_cached_system_content_normalizes_mixed_blocks(self): + """A cached block keeps its marker, plain strings beside it become text blocks, empties drop.""" + from litellm.llms.sap.chat.transformation import _messages_to_sap_template + + ephemeral = {"type": "ephemeral"} + messages = [ + { + "role": "system", + "content": [ + "raw string", + {"type": "text", "text": ""}, + {"type": "text", "text": "B", "cache_control": ephemeral}, + ], + } + ] + + content = _messages_to_sap_template(messages)[0]["content"] + + assert list(content) == [ + {"type": "text", "text": "raw string"}, + {"type": "text", "text": "B", "cache_control": ephemeral}, + ] + + def test_assistant_and_tool_messages_still_flatten(self): + """SAP's own SDK skips assistant and tool roles when applying cache_control, so they stay strings.""" + from litellm.llms.sap.chat.transformation import _messages_to_sap_template + + ephemeral = {"type": "ephemeral"} + messages = [ + {"role": "assistant", "content": [{"type": "text", "text": "ans", "cache_control": ephemeral}]}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "res", "cache_control": ephemeral}], + }, + ] + + template = _messages_to_sap_template(messages) + + assert template[0]["content"] == "ans" + assert template[1]["content"] == "res"