From 808bad989dbc69b05314ef60f9a2986162519fce Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Tue, 20 Oct 2026 12:55:06 +0200 Subject: [PATCH] fix(sap): normalize reasoning content --- litellm/llms/sap/chat/handler.py | 79 +++++-- litellm/llms/sap/chat/models.py | 6 + litellm/llms/sap/chat/transformation.py | 3 +- .../llms/sap/chat/test_sap_transformation.py | 219 ++++++++++++++++++ 4 files changed, 287 insertions(+), 20 deletions(-) diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index b4f4d4faddb..0866548ce7f 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -23,15 +23,54 @@ class GenAIHubOrchestrationError(BaseLLMException): self.message = message -# ------------------------------- -# Stream parsing helpers -# ------------------------------- +# ---------------------------------- +# Parsing and transformation helpers +# ---------------------------------- def _now_ts() -> int: return int(time.time()) +def normalize_reasoning_content(raw: dict[str, object]) -> dict[str, object]: + # SAP AI Core returns reasoning tokens as a list: message.reasoning_content = [{"content": ..., "signature": ...}] + # ModelResponse.reasoning_content is typed Optional[str], so map the list to + # thinking_blocks and collapse the text into a single joined string. + choices: Final[list[object]] = list(raw.get("choices") or []) + normalized_choices: Final[dict[str, object]] = [ + normalize_choice(choice) for choice in choices if isinstance(choice, dict) + ] + normalized_result: Final[dict[str, object]] = {**raw, "choices": normalized_choices} + return normalized_result + + +def normalize_choice(choice: dict[str, object]) -> dict[str, object]: + # Normalize a single choice dict (message or delta shape) in isolation. + # Exposed as a static method so tests and callers can exercise one choice + # without constructing a full response envelope. + for key in ("message", "delta"): + carrier = choice.get(key) + if not isinstance(carrier, dict): + continue + reasoning_content = carrier.get("reasoning_content") + if not isinstance(reasoning_content, list): + continue + thinking_blocks = [ + { + "type": "thinking", + "thinking": item.get("content") or "", + "signature": item.get("signature"), + } + for item in reasoning_content + if isinstance(item, dict) + ] + carrier["thinking_blocks"] = thinking_blocks + carrier["reasoning_content"] = ( + "\n".join(block["thinking"] for block in thinking_blocks if block["thinking"]) or None + ) + return choice + + def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool: """OpenAI-shaped chunk is terminal if any choice has a non-None finish_reason.""" try: @@ -56,20 +95,22 @@ class _StreamParser: return None return OpenAIChatCompletionChunk.model_validate( - { - "id": orc.get("id") or evt.get("request_id") or "stream-chunk", - "object": orc.get("object") or "chat.completion.chunk", - "created": orc.get("created") or evt.get("created") or _now_ts(), - "model": orc.get("model") or "unknown", - "choices": [ - { - "index": c.get("index", 0), - "delta": c.get("delta") or {}, - "finish_reason": c.get("finish_reason"), - } - for c in (orc.get("choices") or []) - ], - } + normalize_reasoning_content( + { + "id": orc.get("id") or evt.get("request_id") or "stream-chunk", + "object": orc.get("object") or "chat.completion.chunk", + "created": orc.get("created") or evt.get("created") or _now_ts(), + "model": orc.get("model") or "unknown", + "choices": [ + { + "index": choice.get("index", 0), + "delta": choice.get("delta") or {}, + "finish_reason": choice.get("finish_reason"), + } + for choice in (orc.get("choices") or []) + ], + } + ) ) @staticmethod @@ -93,7 +134,7 @@ class _StreamParser: # ensure it looks like an OpenAI chunk if "object" not in fr: fr["object"] = "chat.completion.chunk" - return OpenAIChatCompletionChunk.model_validate(fr) + return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(fr)) # Orchestration incremental delta if "orchestration_result" in event_obj: @@ -101,7 +142,7 @@ class _StreamParser: # Already an OpenAI-like chunk if "choices" in event_obj and "object" in event_obj: - return OpenAIChatCompletionChunk.model_validate(event_obj) + return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(event_obj)) # Unknown / heartbeat / metrics return None diff --git a/litellm/llms/sap/chat/models.py b/litellm/llms/sap/chat/models.py index 43fa3236e88..b232eef23da 100644 --- a/litellm/llms/sap/chat/models.py +++ b/litellm/llms/sap/chat/models.py @@ -124,11 +124,17 @@ class SAPUserMessage(LiteLLMBaseModel): content: str | TextContent | ImageContent | list[TextContent | ImageContent] +class ReasoningBlock(LiteLLMBaseModel): + content: str | None = None + signature: str | None = None + + class SAPAssistantMessage(LiteLLMBaseModel): role: Literal["assistant"] = "assistant" content: str | TextContent | list[TextContent] = "" refusal: str = "" tool_calls: list[MessageToolCall] = Field(default=[]) + reasoning_content: list[ReasoningBlock] | None = None class SAPToolChatMessage(LiteLLMBaseModel): diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 34e8acd766a..06e6b1535c6 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -28,6 +28,7 @@ from .handler import ( AsyncSAPStreamIterator, GenAIHubOrchestrationError, SAPStreamIterator, + normalize_reasoning_content, ) from .models import ( ChatCompletionTool, @@ -405,7 +406,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): original_response=raw_response.text, additional_args={"complete_input_dict": request_data}, ) - response = ModelResponse.model_validate(raw_response.json()["final_result"]) + response = ModelResponse.model_validate(normalize_reasoning_content(raw_response.json()["final_result"])) # Strip markdown code blocks if JSON response_format was used with Anthropic models # SAP GenAI Hub with Anthropic models sometimes wraps JSON in ```json ... ``` diff --git a/tests/unit/llms/sap/chat/test_sap_transformation.py b/tests/unit/llms/sap/chat/test_sap_transformation.py index 41e75b7bfc7..e9c87ae8048 100644 --- a/tests/unit/llms/sap/chat/test_sap_transformation.py +++ b/tests/unit/llms/sap/chat/test_sap_transformation.py @@ -923,3 +923,222 @@ class TestCacheControl: content = self._template(body)[0]["content"] assert "cache_control" not in content[0] assert content[1].get("cache_control") == {"type": "ephemeral"} + +class TestNormalizeReasoningContent: + """Unit tests for normalize_reasoning_content.""" + + from litellm.llms.sap.chat.handler import normalize_reasoning_content + + _normalize_reasoning_content = staticmethod(normalize_reasoning_content) + + def test_list_reasoning_content_mapped_to_thinking_blocks(self): + """List-shaped reasoning_content is converted to thinking_blocks.""" + raw = { + "choices": [{ + "message": { + "role": "assistant", + "content": "Latin.", + "reasoning_content": [ + {"content": "Romans spoke Latin.", "signature": "sig1"}, + {"content": "That is well known.", "signature": "sig2"}, + ], + } + }] + } + out = self._normalize_reasoning_content(raw) + msg = out["choices"][0]["message"] + assert msg["thinking_blocks"] == [ + {"type": "thinking", "thinking": "Romans spoke Latin.", "signature": "sig1"}, + {"type": "thinking", "thinking": "That is well known.", "signature": "sig2"}, + ] + assert msg["reasoning_content"] == "Romans spoke Latin.\nThat is well known." + + def test_string_reasoning_content_unchanged(self): + """String reasoning_content is left as-is (already the right type).""" + raw = { + "choices": [{ + "message": { + "role": "assistant", + "content": "42", + "reasoning_content": "I thought about it.", + } + }] + } + out = self._normalize_reasoning_content(raw) + msg = out["choices"][0]["message"] + assert msg["reasoning_content"] == "I thought about it." + assert "thinking_blocks" not in msg + + def test_no_reasoning_content_unchanged(self): + """A message without reasoning_content is not modified.""" + raw = {"choices": [{"message": {"role": "assistant", "content": "Hi."}}]} + out = self._normalize_reasoning_content(raw) + assert out == raw + + def test_empty_list_reasoning_content_sets_none(self): + """An empty list produces None for reasoning_content and empty thinking_blocks.""" + raw = {"choices": [{"message": {"reasoning_content": []}}]} + out = self._normalize_reasoning_content(raw) + msg = out["choices"][0]["message"] + assert msg["thinking_blocks"] == [] + assert msg["reasoning_content"] is None + + def test_multiple_choices_all_normalized(self): + """All choices in the response are normalized.""" + raw = { + "choices": [ + {"message": {"reasoning_content": [{"content": "thought A", "signature": None}]}}, + {"message": {"reasoning_content": [{"content": "thought B", "signature": "s"}]}}, + ] + } + out = self._normalize_reasoning_content(raw) + assert out["choices"][0]["message"]["reasoning_content"] == "thought A" + assert out["choices"][1]["message"]["reasoning_content"] == "thought B" + + + def test_null_content_in_block_uses_empty_string(self): + """Explicit null content value must not leak None into thinking field.""" + raw = { + "choices": [{ + "message": { + "reasoning_content": [{"content": None, "signature": "s"}], + } + }] + } + out = self._normalize_reasoning_content(raw) + block = out["choices"][0]["message"]["thinking_blocks"][0] + assert block["thinking"] == "" + assert out["choices"][0]["message"]["reasoning_content"] is None + + def test_transform_response_normalizes_list_reasoning_content(self): + """Production path: transform_response must produce a ModelResponse + with thinking_blocks populated when the raw payload carries a + list-shaped reasoning_content. + """ + import json + from unittest.mock import MagicMock + + from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig + + config = GenAIHubOrchestrationConfig() + + final_result = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1700000000, + "model": "gemini-test", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "The answer is 42.", + "reasoning_content": [ + {"content": "Let me think.", "signature": "sig1"}, + {"content": "Yes, 42.", "signature": "sig2"}, + ], + }, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + + raw_response = MagicMock() + raw_response.text = json.dumps({"final_result": final_result}) + raw_response.json.return_value = {"final_result": final_result} + + response = config.transform_response( + model="gemini-test", + raw_response=raw_response, + model_response=MagicMock(), + logging_obj=MagicMock(), + api_key="test", + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + choice = response.choices[0] + assert hasattr(choice.message, "thinking_blocks"), "thinking_blocks missing from message" + assert choice.message.thinking_blocks == [ + {"type": "thinking", "thinking": "Let me think.", "signature": "sig1"}, + {"type": "thinking", "thinking": "Yes, 42.", "signature": "sig2"}, + ] + assert choice.message.reasoning_content == "Let me think.\nYes, 42." + + +class TestNormalizeChoice: + """Unit tests for normalize_choice (message and delta shapes).""" + + from litellm.llms.sap.chat.handler import normalize_choice, normalize_reasoning_content + + _normalize_choice = staticmethod(normalize_choice) + _normalize_reasoning_content = staticmethod(normalize_reasoning_content) + + def test_message_list_reasoning_content_normalized(self): + choice = { + "index": 0, + "message": {"role": "assistant", "content": "hi", "reasoning_content": [{"content": "thought", "signature": "s1"}]}, + "finish_reason": "stop", + } + result = self._normalize_choice(choice) + msg = result["message"] + assert msg["reasoning_content"] == "thought" + assert msg["thinking_blocks"] == [{"type": "thinking", "thinking": "thought", "signature": "s1"}] + + def test_delta_list_reasoning_content_normalized(self): + choice = { + "index": 0, + "delta": {"role": "assistant", "reasoning_content": [{"content": "delta thought", "signature": None}]}, + "finish_reason": None, + } + result = self._normalize_choice(choice) + delta = result["delta"] + assert delta["reasoning_content"] == "delta thought" + assert delta["thinking_blocks"][0]["thinking"] == "delta thought" + + def test_string_reasoning_content_unchanged(self): + choice = {"index": 0, "message": {"reasoning_content": "already a string"}} + assert self._normalize_choice(choice) == choice + + def test_no_reasoning_content_unchanged(self): + choice = {"index": 0, "delta": {"content": "hello"}} + assert self._normalize_choice(choice) == choice + + def test_normalize_reasoning_content_covers_delta_path(self): + raw = { + "id": "c1", + "object": "chat.completion.chunk", + "choices": [ + {"index": 0, "delta": {"reasoning_content": [{"content": "stream thought", "signature": "sig"}]}, "finish_reason": None} + ], + } + result = self._normalize_reasoning_content(raw) + delta = result["choices"][0]["delta"] + assert delta["reasoning_content"] == "stream thought" + assert delta["thinking_blocks"][0]["type"] == "thinking" + + +class TestMessagesToSapTemplateReasoningContent: + """_messages_to_sap_template must forward reasoning_content on assistant turns.""" + + def test_assistant_message_with_reasoning_content_is_preserved(self): + from litellm.llms.sap.chat.transformation import _messages_to_sap_template + + messages = [ + {"role": "user", "content": "What is 22?"}, + { + "role": "assistant", + "content": "4", + "reasoning_content": [ + {"content": "Simple arithmetic.", "signature": "sig1"} + ], + }, + {"role": "user", "content": "Are you sure?"}, + ] + result = _messages_to_sap_template(messages) + assistant_msg = result[1] + assert assistant_msg["reasoning_content"] == [ + {"content": "Simple arithmetic.", "signature": "sig1"} + ] \ No newline at end of file