scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 267

fix(sap): normalize reasoning content

Browse source
This commit is contained in:
Yamac Ay 2026-10-20 12:55:06 +02:00
parent de0ed1557e
commit 808bad989d
No known key found for this signature in database
GPG key ID: D113B438819CA628
4 changed files with 287 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
litellm/llms/sap/chat/handler.py
View file

@ -23,15 +23,54 @@ class GenAIHubOrchestrationError(BaseLLMException):
self.message = message
# -------------------------------
# Stream parsing helpers
# -------------------------------
# ----------------------------------
# Parsing and transformation helpers
# ----------------------------------
def _now_ts() -> int:
return int(time.time())
def normalize_reasoning_content(raw: dict[str, object]) -> dict[str, object]:
# SAP AI Core returns reasoning tokens as a list: message.reasoning_content = [{"content": ..., "signature": ...}]
# ModelResponse.reasoning_content is typed Optional[str], so map the list to
# thinking_blocks and collapse the text into a single joined string.
choices: Final[list[object]] = list(raw.get("choices") or [])
normalized_choices: Final[dict[str, object]] = [
normalize_choice(choice) for choice in choices if isinstance(choice, dict)
]
normalized_result: Final[dict[str, object]] = {**raw, "choices": normalized_choices}
return normalized_result
def normalize_choice(choice: dict[str, object]) -> dict[str, object]:
# Normalize a single choice dict (message or delta shape) in isolation.
# Exposed as a static method so tests and callers can exercise one choice
# without constructing a full response envelope.
for key in ("message", "delta"):
carrier = choice.get(key)
if not isinstance(carrier, dict):
continue
reasoning_content = carrier.get("reasoning_content")
if not isinstance(reasoning_content, list):
continue
thinking_blocks = [
{
"type": "thinking",
"thinking": item.get("content") or "",
"signature": item.get("signature"),
}
for item in reasoning_content
if isinstance(item, dict)
]
carrier["thinking_blocks"] = thinking_blocks
carrier["reasoning_content"] = (
"\n".join(block["thinking"] for block in thinking_blocks if block["thinking"]) or None
)
return choice
def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool:
"""OpenAI-shaped chunk is terminal if any choice has a non-None finish_reason."""
try:
@ -56,20 +95,22 @@ class _StreamParser:
return None
return OpenAIChatCompletionChunk.model_validate(
{
"id": orc.get("id") or evt.get("request_id") or "stream-chunk",
"object": orc.get("object") or "chat.completion.chunk",
"created": orc.get("created") or evt.get("created") or _now_ts(),
"model": orc.get("model") or "unknown",
"choices": [
{
"index": c.get("index", 0),
"delta": c.get("delta") or {},
"finish_reason": c.get("finish_reason"),
}
for c in (orc.get("choices") or [])
],
}
normalize_reasoning_content(
{
"id": orc.get("id") or evt.get("request_id") or "stream-chunk",
"object": orc.get("object") or "chat.completion.chunk",
"created": orc.get("created") or evt.get("created") or _now_ts(),
"model": orc.get("model") or "unknown",
"choices": [
{
"index": choice.get("index", 0),
"delta": choice.get("delta") or {},
"finish_reason": choice.get("finish_reason"),
}
for choice in (orc.get("choices") or [])
],
}
)
)
@staticmethod
@ -93,7 +134,7 @@ class _StreamParser:
# ensure it looks like an OpenAI chunk
if "object" not in fr:
fr["object"] = "chat.completion.chunk"
return OpenAIChatCompletionChunk.model_validate(fr)
return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(fr))
# Orchestration incremental delta
if "orchestration_result" in event_obj:
@ -101,7 +142,7 @@ class _StreamParser:
# Already an OpenAI-like chunk
if "choices" in event_obj and "object" in event_obj:
return OpenAIChatCompletionChunk.model_validate(event_obj)
return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(event_obj))
# Unknown / heartbeat / metrics
return None

6
litellm/llms/sap/chat/models.py
View file

@ -124,11 +124,17 @@ class SAPUserMessage(LiteLLMBaseModel):
content: str | TextContent | ImageContent | list[TextContent | ImageContent]
class ReasoningBlock(LiteLLMBaseModel):
content: str | None = None
signature: str | None = None
class SAPAssistantMessage(LiteLLMBaseModel):
role: Literal["assistant"] = "assistant"
content: str | TextContent | list[TextContent] = ""
refusal: str = ""
tool_calls: list[MessageToolCall] = Field(default=[])
reasoning_content: list[ReasoningBlock] | None = None
class SAPToolChatMessage(LiteLLMBaseModel):

3
litellm/llms/sap/chat/transformation.py
View file

@ -28,6 +28,7 @@ from .handler import (
AsyncSAPStreamIterator,
GenAIHubOrchestrationError,
SAPStreamIterator,
normalize_reasoning_content,
)
from .models import (
ChatCompletionTool,
@ -405,7 +406,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
original_response=raw_response.text,
additional_args={"complete_input_dict": request_data},
)
response = ModelResponse.model_validate(raw_response.json()["final_result"])
response = ModelResponse.model_validate(normalize_reasoning_content(raw_response.json()["final_result"]))
# Strip markdown code blocks if JSON response_format was used with Anthropic models
# SAP GenAI Hub with Anthropic models sometimes wraps JSON in ```json ... ```

219
tests/unit/llms/sap/chat/test_sap_transformation.py
View file

@ -923,3 +923,222 @@ class TestCacheControl:
content = self._template(body)[0]["content"]
assert "cache_control" not in content[0]
assert content[1].get("cache_control") == {"type": "ephemeral"}
class TestNormalizeReasoningContent:
"""Unit tests for normalize_reasoning_content."""
from litellm.llms.sap.chat.handler import normalize_reasoning_content
_normalize_reasoning_content = staticmethod(normalize_reasoning_content)
def test_list_reasoning_content_mapped_to_thinking_blocks(self):
"""List-shaped reasoning_content is converted to thinking_blocks."""
raw = {
"choices": [{
"message": {
"role": "assistant",
"content": "Latin.",
"reasoning_content": [
{"content": "Romans spoke Latin.", "signature": "sig1"},
{"content": "That is well known.", "signature": "sig2"},
],
}
}]
}
out = self._normalize_reasoning_content(raw)
msg = out["choices"][0]["message"]
assert msg["thinking_blocks"] == [
{"type": "thinking", "thinking": "Romans spoke Latin.", "signature": "sig1"},
{"type": "thinking", "thinking": "That is well known.", "signature": "sig2"},
]
assert msg["reasoning_content"] == "Romans spoke Latin.\nThat is well known."
def test_string_reasoning_content_unchanged(self):
"""String reasoning_content is left as-is (already the right type)."""
raw = {
"choices": [{
"message": {
"role": "assistant",
"content": "42",
"reasoning_content": "I thought about it.",
}
}]
}
out = self._normalize_reasoning_content(raw)
msg = out["choices"][0]["message"]
assert msg["reasoning_content"] == "I thought about it."
assert "thinking_blocks" not in msg
def test_no_reasoning_content_unchanged(self):
"""A message without reasoning_content is not modified."""
raw = {"choices": [{"message": {"role": "assistant", "content": "Hi."}}]}
out = self._normalize_reasoning_content(raw)
assert out == raw
def test_empty_list_reasoning_content_sets_none(self):
"""An empty list produces None for reasoning_content and empty thinking_blocks."""
raw = {"choices": [{"message": {"reasoning_content": []}}]}
out = self._normalize_reasoning_content(raw)
msg = out["choices"][0]["message"]
assert msg["thinking_blocks"] == []
assert msg["reasoning_content"] is None
def test_multiple_choices_all_normalized(self):
"""All choices in the response are normalized."""
raw = {
"choices": [
{"message": {"reasoning_content": [{"content": "thought A", "signature": None}]}},
{"message": {"reasoning_content": [{"content": "thought B", "signature": "s"}]}},
]
}
out = self._normalize_reasoning_content(raw)
assert out["choices"][0]["message"]["reasoning_content"] == "thought A"
assert out["choices"][1]["message"]["reasoning_content"] == "thought B"
def test_null_content_in_block_uses_empty_string(self):
"""Explicit null content value must not leak None into thinking field."""
raw = {
"choices": [{
"message": {
"reasoning_content": [{"content": None, "signature": "s"}],
}
}]
}
out = self._normalize_reasoning_content(raw)
block = out["choices"][0]["message"]["thinking_blocks"][0]
assert block["thinking"] == ""
assert out["choices"][0]["message"]["reasoning_content"] is None
def test_transform_response_normalizes_list_reasoning_content(self):
"""Production path: transform_response must produce a ModelResponse
with thinking_blocks populated when the raw payload carries a
list-shaped reasoning_content.
"""
import json
from unittest.mock import MagicMock
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
config = GenAIHubOrchestrationConfig()
final_result = {
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 1700000000,
"model": "gemini-test",
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": "The answer is 42.",
"reasoning_content": [
{"content": "Let me think.", "signature": "sig1"},
{"content": "Yes, 42.", "signature": "sig2"},
],
},
"finish_reason": "stop",
}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
raw_response = MagicMock()
raw_response.text = json.dumps({"final_result": final_result})
raw_response.json.return_value = {"final_result": final_result}
response = config.transform_response(
model="gemini-test",
raw_response=raw_response,
model_response=MagicMock(),
logging_obj=MagicMock(),
api_key="test",
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=None,
)
choice = response.choices[0]
assert hasattr(choice.message, "thinking_blocks"), "thinking_blocks missing from message"
assert choice.message.thinking_blocks == [
{"type": "thinking", "thinking": "Let me think.", "signature": "sig1"},
{"type": "thinking", "thinking": "Yes, 42.", "signature": "sig2"},
]
assert choice.message.reasoning_content == "Let me think.\nYes, 42."
class TestNormalizeChoice:
"""Unit tests for normalize_choice (message and delta shapes)."""
from litellm.llms.sap.chat.handler import normalize_choice, normalize_reasoning_content
_normalize_choice = staticmethod(normalize_choice)
_normalize_reasoning_content = staticmethod(normalize_reasoning_content)
def test_message_list_reasoning_content_normalized(self):
choice = {
"index": 0,
"message": {"role": "assistant", "content": "hi", "reasoning_content": [{"content": "thought", "signature": "s1"}]},
"finish_reason": "stop",
}
result = self._normalize_choice(choice)
msg = result["message"]
assert msg["reasoning_content"] == "thought"
assert msg["thinking_blocks"] == [{"type": "thinking", "thinking": "thought", "signature": "s1"}]
def test_delta_list_reasoning_content_normalized(self):
choice = {
"index": 0,
"delta": {"role": "assistant", "reasoning_content": [{"content": "delta thought", "signature": None}]},
"finish_reason": None,
}
result = self._normalize_choice(choice)
delta = result["delta"]
assert delta["reasoning_content"] == "delta thought"
assert delta["thinking_blocks"][0]["thinking"] == "delta thought"
def test_string_reasoning_content_unchanged(self):
choice = {"index": 0, "message": {"reasoning_content": "already a string"}}
assert self._normalize_choice(choice) == choice
def test_no_reasoning_content_unchanged(self):
choice = {"index": 0, "delta": {"content": "hello"}}
assert self._normalize_choice(choice) == choice
def test_normalize_reasoning_content_covers_delta_path(self):
raw = {
"id": "c1",
"object": "chat.completion.chunk",
"choices": [
{"index": 0, "delta": {"reasoning_content": [{"content": "stream thought", "signature": "sig"}]}, "finish_reason": None}
],
}
result = self._normalize_reasoning_content(raw)
delta = result["choices"][0]["delta"]
assert delta["reasoning_content"] == "stream thought"
assert delta["thinking_blocks"][0]["type"] == "thinking"
class TestMessagesToSapTemplateReasoningContent:
"""_messages_to_sap_template must forward reasoning_content on assistant turns."""
def test_assistant_message_with_reasoning_content_is_preserved(self):
from litellm.llms.sap.chat.transformation import _messages_to_sap_template
messages = [
{"role": "user", "content": "What is 22?"},
{
"role": "assistant",
"content": "4",
"reasoning_content": [
{"content": "Simple arithmetic.", "signature": "sig1"}
],
},
{"role": "user", "content": "Are you sure?"},
]
result = _messages_to_sap_template(messages)
assistant_msg = result[1]
assert assistant_msg["reasoning_content"] == [
{"content": "Simple arithmetic.", "signature": "sig1"}
]