This commit is contained in:
Yamac Eren Ay 2026-11-05 03:24:43 +00:00 • committed by GitHub
commit a7b84b5939
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 480 additions and 32 deletions

View file

@ -23,15 +23,100 @@ class GenAIHubOrchestrationError(BaseLLMException):
self.message = message
# -------------------------------
# Stream parsing helpers
# -------------------------------
# ----------------------------------
# Parsing and transformation helpers
# ----------------------------------
def _now_ts() -> int:
return int(time.time())
def normalize_choice(
choice: dict[str, object], # mutable-ok: wire dicts from httpx JSON; callers own them
) -> dict[str, object]: # mutable-ok: wire dicts from httpx JSON; callers own them
"""Normalize a single choice dict (message or delta shape) in isolation."""
for key in ("message", "delta"):
carrier = choice.get(key)
if not isinstance(carrier, dict):
continue
reasoning_content = carrier.get("reasoning_content")
if not isinstance(reasoning_content, list):
continue
thinking_blocks: list[dict[str, object]] = [ # mutable-ok: consumed immediately below; never escapes as mutable
{
"type": "thinking",
"thinking": item.get("content") or "", # pyright: ignore[reportUnknownMemberType] # isinstance guard in comprehension filter; pyright can't narrow
"signature": item.get( # pyright: ignore[reportUnknownMemberType] # isinstance guard in comprehension filter; pyright can't narrow
"signature"
),
}
for item in reasoning_content # pyright: ignore[reportUnknownVariableType] # isinstance(reasoning_content, list) guard above; pyright can't narrow into comprehension
if isinstance(item, dict)
]
carrier["thinking_blocks"] = thinking_blocks
carrier["reasoning_content"] = (
"\n".join(str(block["thinking"]) for block in thinking_blocks if block["thinking"]) or None
)
return choice
def normalize_reasoning_content(
raw: dict[str, object], # mutable-ok: wire dicts from httpx JSON; callers own them
) -> dict[str, object]: # mutable-ok: wire dicts from httpx JSON; callers own them
"""Transform reasoning_content from SAP Orchestration format [{"content": ..., "signature": ...}] to Optional[str] and a thinking_blocks list."""
choices: Final[list[object]] = list( # mutable-ok: wire list from httpx JSON
raw.get("choices") or [] # pyright: ignore[reportArgumentType, reportUnknownArgumentType] # wire JSON; choices is always a list
)
normalized_choices: Final[list[dict[str, object]]] = [ # mutable-ok: wire list passed to pydantic model_validate
normalize_choice(choice) # pyright: ignore[reportUnknownArgumentType] # isinstance guard in comprehension filter; pyright can't narrow
for choice in choices
if isinstance(choice, dict)
]
normalized_result: Final[dict[str, object]] = { # mutable-ok: wire dict consumed by pydantic model_validate
**raw,
"choices": normalized_choices,
}
return normalized_result
def denormalize_reasoning_content(
message: dict[str, object], # mutable-ok: wire dicts from httpx JSON; callers own them
) -> dict[str, object]: # mutable-ok: wire dicts from httpx JSON; callers own them
"""Inverse of normalize_reasoning_content: rebuild list[ReasoningBlock] from thinking_blocks.
LiteLLM normalizes SAP's reasoning_content list into a str + thinking_blocks before
returning a ModelResponse. When that response is replayed as a history message the dict
must be converted back to the SAP shape before SAPAssistantMessage validation.
If thinking_blocks is absent but reasoning_content is a plain str (e.g. from a
non-SAP provider), drop the field: we cannot reconstruct SAP ReasoningBlocks without
the original signatures.
"""
if message.get("role") != "assistant":
return message
thinking_blocks: Final = message.get("thinking_blocks")
if isinstance(thinking_blocks, list):
reasoning_content: Final[list[dict[str, str | None]]] = [ # mutable-ok: wire list passed to validate_dict
{
"content": str(
block.get("thinking") or "" # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # isinstance guard in comprehension filter; pyright can't narrow
),
"signature": block.get( # pyright: ignore[reportUnknownMemberType] # isinstance guard in comprehension filter; pyright can't narrow
"signature"
),
}
for block in thinking_blocks # pyright: ignore[reportUnknownVariableType] # isinstance(thinking_blocks, list) guard above; pyright can't narrow into comprehension
if isinstance(block, dict)
]
return {k: v for k, v in message.items() if k not in ("thinking_blocks", "reasoning_content")} | {
"reasoning_content": reasoning_content
}
if isinstance(message.get("reasoning_content"), str):
return {k: v for k, v in message.items() if k not in ("thinking_blocks", "reasoning_content")}
return message
def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool:
"""OpenAI-shaped chunk is terminal if any choice has a non-None finish_reason."""
try:
@ -47,33 +132,37 @@ class _StreamParser:
"""Normalize orchestration streaming events into OpenAI-like chunks."""
@staticmethod
def _from_orchestration_result(evt: dict) -> OpenAIChatCompletionChunk | None:
def _from_orchestration_result(evt: dict[str, object]) -> OpenAIChatCompletionChunk | None:
"""
Accepts orchestration_result shape and maps it to an OpenAI-like *chunk*.
"""
orc: Final = evt.get("orchestration_result") or {}
orc: Final[dict[str, object]] = evt.get("orchestration_result") or {} # pyright: ignore[reportAssignmentType] # wire JSON; orchestration_result is always a dict
if not orc:
return None
return OpenAIChatCompletionChunk.model_validate(
{
"id": orc.get("id") or evt.get("request_id") or "stream-chunk",
"object": orc.get("object") or "chat.completion.chunk",
"created": orc.get("created") or evt.get("created") or _now_ts(),
"model": orc.get("model") or "unknown",
"choices": [
{
"index": c.get("index", 0),
"delta": c.get("delta") or {},
"finish_reason": c.get("finish_reason"),
}
for c in (orc.get("choices") or [])
],
}
normalize_reasoning_content(
{
"id": orc.get("id")
or evt.get("request_id") # pyright: ignore[reportUnknownMemberType] # wire JSON string field
or "stream-chunk",
"object": orc.get("object") or "chat.completion.chunk",
"created": orc.get("created") or evt.get("created") or _now_ts(),
"model": orc.get("model") or "unknown",
"choices": [
{
"index": choice.get("index", 0),
"delta": choice.get("delta") or {},
"finish_reason": choice.get("finish_reason"),
}
for choice in orc.get("choices") or [] # pyright: ignore[reportUnknownVariableType, reportGeneralTypeIssues] # wire JSON; choices is always list[dict]
],
}
)
)
@staticmethod
def to_openai_chunk(event_obj: dict) -> OpenAIChatCompletionChunk | None:
def to_openai_chunk(event_obj: dict[str, object]) -> OpenAIChatCompletionChunk | None:
"""
Accepts:
- {"final_result": <openai-style CHUNK>} (IMPORTANT: this is just another chunk, NOT terminal)
@ -89,11 +178,14 @@ class _StreamParser:
# FINAL RESULT IS *NOT* TERMINAL: treat it as the next chunk
if "final_result" in event_obj:
fr: Final = event_obj["final_result"] or {}
# ensure it looks like an OpenAI chunk
if "object" not in fr:
fr["object"] = "chat.completion.chunk"
return OpenAIChatCompletionChunk.model_validate(fr)
fr_raw: Final = event_obj["final_result"] or {}
if not isinstance(fr_raw, dict):
return None
fr: Final[dict[str, object]] = { # mutable-ok: wire dict consumed by model_validate
**fr_raw,
**(({"object": "chat.completion.chunk"}) if "object" not in fr_raw else {}),
}
return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(fr))
# Orchestration incremental delta
if "orchestration_result" in event_obj:
@ -101,7 +193,7 @@ class _StreamParser:
# Already an OpenAI-like chunk
if "choices" in event_obj and "object" in event_obj:
return OpenAIChatCompletionChunk.model_validate(event_obj)
return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(event_obj))
# Unknown / heartbeat / metrics
return None
@ -248,7 +340,8 @@ class AsyncSAPStreamIterator:
# -------------------------------
class GenAIHubOrchestration(BaseLLMHTTPHandler):
def _add_stream_param_to_request_body(self, data: dict, provider_config: BaseConfig, fake_stream: bool):
if data.get("config", {}).get("stream", None) is not None:
config = data.get("config") or {} # pyright: ignore[reportUnknownVariableType] # wire JSON; config is always a dict when present
if isinstance(config, dict) and config.get("stream") is not None:
data["config"]["stream"]["enabled"] = True
else:
data["config"]["stream"] = {"enabled": True}

View file

@ -133,11 +133,21 @@ class SAPUserMessage(LiteLLMBaseModel):
content: str | TextContent | ImageContent | list[TextContent | ImageContent]
class ReasoningBlock( # frozen-ok: adding model_config would override LiteLLMBaseModel's defer_build setting
LiteLLMBaseModel
):
content: str | None = None
signature: str | None = None
class SAPAssistantMessage(LiteLLMBaseModel):
role: Literal["assistant"] = "assistant"
content: str | TextContent | list[TextContent] = ""
refusal: str = ""
tool_calls: list[MessageToolCall] = Field(default=[])
tool_calls: list[MessageToolCall] = Field( # mutable-ok: pydantic field; callers append tool calls to this list
default=[]
)
reasoning_content: list[ReasoningBlock] | None = None # mutable-ok: list of reasoning blocks from SAP wire format
class SAPToolChatMessage(LiteLLMBaseModel):

View file

@ -31,6 +31,8 @@ from .handler import (
AsyncSAPStreamIterator,
GenAIHubOrchestrationError,
SAPStreamIterator,
denormalize_reasoning_content,
normalize_reasoning_content,
)
from .models import (
ChatCompletionTool,
@ -106,8 +108,9 @@ def _messages_to_sap_template(messages: list[AllMessageValues]) -> list:
template: Final = []
for message in messages:
folded_message = _fold_message_cache_control(message_with_file_parts_as_image_urls(message))
message_role_class = _message_role_mapping(str(folded_message["role"]))
validated_message = validate_dict(folded_message, message_role_class)
denormalized_message = denormalize_reasoning_content(dict(folded_message))
message_role_class = _message_role_mapping(str(denormalized_message["role"]))
validated_message = validate_dict(denormalized_message, message_role_class)
template.append(validated_message)
return template
@ -229,7 +232,9 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
"functions",
"extra_headers",
"parallel_tool_calls",
"reasoning_effort",
"response_format",
"thinking",
"timeout",
]
# Remove response_format for providers that don't support it on SAP GenAI Hub
@ -410,7 +415,9 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
original_response=raw_response.text,
additional_args={"complete_input_dict": request_data},
)
response = ModelResponse.model_validate(raw_response.json()["final_result"])
response: Final[ModelResponse] = ModelResponse.model_validate(
normalize_reasoning_content(raw_response.json()["final_result"])
)
# Strip markdown code blocks if JSON response_format was used with Anthropic models
# SAP GenAI Hub with Anthropic models sometimes wraps JSON in ```json ... ```
@ -419,7 +426,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
response_format: Final = optional_params.get("response_format", {})
if response_format.get("type") in ("json_object", "json_schema"):
if model.startswith("anthropic"):
response = self._strip_markdown_json(response)
return self._strip_markdown_json(response)
return response

View file

@ -941,3 +941,341 @@ class TestCacheControl:
content = self._template(body)[0]["content"]
assert "cache_control" not in content[0]
assert content[1].get("cache_control") == {"type": "ephemeral"}
class TestNormalizeReasoningContent:
"""Unit tests for normalize_reasoning_content."""
from litellm.llms.sap.chat.handler import normalize_reasoning_content
_normalize_reasoning_content = staticmethod(normalize_reasoning_content)
def test_list_reasoning_content_mapped_to_thinking_blocks(self):
"""List-shaped reasoning_content is converted to thinking_blocks."""
raw = {
"choices": [{
"message": {
"role": "assistant",
"content": "Latin.",
"reasoning_content": [
{"content": "Romans spoke Latin.", "signature": "sig1"},
{"content": "That is well known.", "signature": "sig2"},
],
}
}]
}
out = self._normalize_reasoning_content(raw)
msg = out["choices"][0]["message"]
assert msg["thinking_blocks"] == [
{"type": "thinking", "thinking": "Romans spoke Latin.", "signature": "sig1"},
{"type": "thinking", "thinking": "That is well known.", "signature": "sig2"},
]
assert msg["reasoning_content"] == "Romans spoke Latin.\nThat is well known."
def test_string_reasoning_content_unchanged(self):
"""String reasoning_content is left as-is (already the right type)."""
raw = {
"choices": [{
"message": {
"role": "assistant",
"content": "42",
"reasoning_content": "I thought about it.",
}
}]
}
out = self._normalize_reasoning_content(raw)
msg = out["choices"][0]["message"]
assert msg["reasoning_content"] == "I thought about it."
assert "thinking_blocks" not in msg
def test_no_reasoning_content_unchanged(self):
"""A message without reasoning_content is not modified."""
raw = {"choices": [{"message": {"role": "assistant", "content": "Hi."}}]}
out = self._normalize_reasoning_content(raw)
assert out == raw
def test_empty_list_reasoning_content_sets_none(self):
"""An empty list produces None for reasoning_content and empty thinking_blocks."""
raw = {"choices": [{"message": {"reasoning_content": []}}]}
out = self._normalize_reasoning_content(raw)
msg = out["choices"][0]["message"]
assert msg["thinking_blocks"] == []
assert msg["reasoning_content"] is None
def test_multiple_choices_all_normalized(self):
"""All choices in the response are normalized."""
raw = {
"choices": [
{"message": {"reasoning_content": [{"content": "thought A", "signature": None}]}},
{"message": {"reasoning_content": [{"content": "thought B", "signature": "s"}]}},
]
}
out = self._normalize_reasoning_content(raw)
assert out["choices"][0]["message"]["reasoning_content"] == "thought A"
assert out["choices"][1]["message"]["reasoning_content"] == "thought B"
def test_null_content_in_block_uses_empty_string(self):
"""Explicit null content value must not leak None into thinking field."""
raw = {
"choices": [{
"message": {
"reasoning_content": [{"content": None, "signature": "s"}],
}
}]
}
out = self._normalize_reasoning_content(raw)
block = out["choices"][0]["message"]["thinking_blocks"][0]
assert block["thinking"] == ""
assert out["choices"][0]["message"]["reasoning_content"] is None
def test_transform_response_normalizes_list_reasoning_content(self):
"""Production path: transform_response must produce a ModelResponse
with thinking_blocks populated when the raw payload carries a
list-shaped reasoning_content.
"""
import json
from unittest.mock import MagicMock
from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
config = GenAIHubOrchestrationConfig()
final_result = {
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 1700000000,
"model": "gemini-test",
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": "The answer is 42.",
"reasoning_content": [
{"content": "Let me think.", "signature": "sig1"},
{"content": "Yes, 42.", "signature": "sig2"},
],
},
"finish_reason": "stop",
}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
raw_response = MagicMock()
raw_response.text = json.dumps({"final_result": final_result})
raw_response.json.return_value = {"final_result": final_result}
response = config.transform_response(
model="gemini-test",
raw_response=raw_response,
model_response=MagicMock(),
logging_obj=MagicMock(),
api_key="test",
request_data={},
messages=[],
optional_params={},
litellm_params={},
encoding=None,
)
choice = response.choices[0]
assert hasattr(choice.message, "thinking_blocks"), "thinking_blocks missing from message"
assert choice.message.thinking_blocks == [
{"type": "thinking", "thinking": "Let me think.", "signature": "sig1"},
{"type": "thinking", "thinking": "Yes, 42.", "signature": "sig2"},
]
assert choice.message.reasoning_content == "Let me think.\nYes, 42."
class TestNormalizeChoice:
"""Unit tests for normalize_choice (message and delta shapes)."""
from litellm.llms.sap.chat.handler import normalize_choice, normalize_reasoning_content
_normalize_choice = staticmethod(normalize_choice)
_normalize_reasoning_content = staticmethod(normalize_reasoning_content)
def test_message_list_reasoning_content_normalized(self):
choice = {
"index": 0,
"message": {"role": "assistant", "content": "hi", "reasoning_content": [{"content": "thought", "signature": "s1"}]},
"finish_reason": "stop",
}
result = self._normalize_choice(choice)
msg = result["message"]
assert msg["reasoning_content"] == "thought"
assert msg["thinking_blocks"] == [{"type": "thinking", "thinking": "thought", "signature": "s1"}]
def test_delta_list_reasoning_content_normalized(self):
choice = {
"index": 0,
"delta": {"role": "assistant", "reasoning_content": [{"content": "delta thought", "signature": None}]},
"finish_reason": None,
}
result = self._normalize_choice(choice)
delta = result["delta"]
assert delta["reasoning_content"] == "delta thought"
assert delta["thinking_blocks"][0]["thinking"] == "delta thought"
def test_string_reasoning_content_unchanged(self):
choice = {"index": 0, "message": {"reasoning_content": "already a string"}}
assert self._normalize_choice(choice) == choice
def test_no_reasoning_content_unchanged(self):
choice = {"index": 0, "delta": {"content": "hello"}}
assert self._normalize_choice(choice) == choice
def test_normalize_reasoning_content_covers_delta_path(self):
raw = {
"id": "c1",
"object": "chat.completion.chunk",
"choices": [
{"index": 0, "delta": {"reasoning_content": [{"content": "stream thought", "signature": "sig"}]}, "finish_reason": None}
],
}
result = self._normalize_reasoning_content(raw)
delta = result["choices"][0]["delta"]
assert delta["reasoning_content"] == "stream thought"
assert delta["thinking_blocks"][0]["type"] == "thinking"
class TestMessagesToSapTemplateReasoningContent:
"""_messages_to_sap_template must forward reasoning_content on assistant turns."""
def test_assistant_message_with_reasoning_content_is_preserved(self):
from litellm.llms.sap.chat.transformation import _messages_to_sap_template
messages = [
{"role": "user", "content": "What is 22?"},
{
"role": "assistant",
"content": "4",
"reasoning_content": [
{"content": "Simple arithmetic.", "signature": "sig1"}
],
},
{"role": "user", "content": "Are you sure?"},
]
result = _messages_to_sap_template(messages)
assistant_msg = result[1]
assert assistant_msg["reasoning_content"] == [
{"content": "Simple arithmetic.", "signature": "sig1"}
]
def test_litellm_normalized_assistant_message_round_trips_signature(self):
"""After a non-streaming SAP call, the ModelResponse has reasoning_content: str
and thinking_blocks: list (LiteLLM native format). Replaying it as a history
message must not raise and must preserve the signature.
"""
from litellm.llms.sap.chat.transformation import _messages_to_sap_template
signature = "EtwHCkABGAIqQKNbQn0Uz=="
messages = [
{"role": "user", "content": "Translate it."},
{
"role": "assistant",
"content": "The text is Lorem Ipsum.",
"reasoning_content": "Lorem ipsum is pseudo-Latin...",
"thinking_blocks": [
{"type": "thinking", "thinking": "Lorem ipsum is pseudo-Latin...", "signature": signature}
],
},
{"role": "user", "content": "Summarise it."},
]
result = _messages_to_sap_template(messages)
assistant = next(m for m in result if m.get("role") == "assistant")
reasoning = assistant.get("reasoning_content")
assert isinstance(reasoning, list)
assert any(block.get("signature") == signature for block in reasoning)
def test_streaming_assembled_assistant_message_round_trips_signature(self):
"""After a streaming SAP call, normalize_reasoning_content runs on every chunk.
The assembled assistant message has the same LiteLLM-native shape as the
non-streaming case. Verify the signature survives the round-trip.
"""
from litellm.llms.sap.chat.handler import normalize_reasoning_content
from litellm.llms.sap.chat.transformation import _messages_to_sap_template
signature = "EtwHCkABGAIqQKNbQn0Uz=="
thinking_text = "Lorem ipsum is pseudo-Latin..."
sap_final_result = {
"id": "msg_stream_01",
"object": "chat.completion",
"model": "gpt-4o",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "The text is Lorem Ipsum.",
"reasoning_content": [{"content": thinking_text, "signature": signature}],
},
"finish_reason": "stop",
}
],
}
normalized = normalize_reasoning_content(sap_final_result)
assembled_message = {"role": "assistant", **normalized["choices"][0]["message"]}
messages = [
{"role": "user", "content": "Translate it."},
assembled_message,
{"role": "user", "content": "Summarize it."},
]
result = _messages_to_sap_template(messages)
assistant = next(m for m in result if m.get("role") == "assistant")
reasoning = assistant.get("reasoning_content")
assert isinstance(reasoning, list)
assert any(block.get("signature") == signature for block in reasoning)
class TestDenormalizeReasoningContent:
from litellm.llms.sap.chat.handler import denormalize_reasoning_content
_denormalize = staticmethod(denormalize_reasoning_content)
def test_thinking_blocks_rebuilt_into_sap_reasoning_content(self):
message = {
"role": "assistant",
"content": "hello",
"thinking_blocks": [{"type": "thinking", "thinking": "step one", "signature": "sig1"}],
"reasoning_content": "step one",
}
result = self._denormalize(message)
assert result["reasoning_content"] == [{"content": "step one", "signature": "sig1"}]
assert "thinking_blocks" not in result
def test_string_reasoning_content_without_thinking_blocks_is_dropped(self):
message = {
"role": "assistant",
"content": "hello",
"reasoning_content": "some thought from another provider",
}
result = self._denormalize(message)
assert "reasoning_content" not in result
assert "thinking_blocks" not in result
assert result["content"] == "hello"
def test_non_assistant_message_is_returned_unchanged(self):
message = {"role": "user", "content": "hi", "reasoning_content": "should stay"}
assert self._denormalize(message) is message
def test_assistant_without_reasoning_fields_is_returned_unchanged(self):
message = {"role": "assistant", "content": "hi"}
assert self._denormalize(message) is message
def test_validate_dict_accepts_denormalized_message_from_string_reasoning_content(self):
from litellm.llms.sap.chat.models import SAPAssistantMessage
from litellm.llms.sap.chat.transformation import validate_dict
message = {
"role": "assistant",
"content": "done",
"reasoning_content": "raw string from openai o-series",
}
result = validate_dict(self._denormalize(message), SAPAssistantMessage)
assert result["role"] == "assistant"
assert "reasoning_content" not in result or result.get("reasoning_content") is None