diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index b4f4d4faddb..21c657d0e3d 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -23,15 +23,100 @@ class GenAIHubOrchestrationError(BaseLLMException): self.message = message -# ------------------------------- -# Stream parsing helpers -# ------------------------------- +# ---------------------------------- +# Parsing and transformation helpers +# ---------------------------------- def _now_ts() -> int: return int(time.time()) +def normalize_choice( + choice: dict[str, object], # mutable-ok: wire dicts from httpx JSON; callers own them +) -> dict[str, object]: # mutable-ok: wire dicts from httpx JSON; callers own them + """Normalize a single choice dict (message or delta shape) in isolation.""" + for key in ("message", "delta"): + carrier = choice.get(key) + if not isinstance(carrier, dict): + continue + reasoning_content = carrier.get("reasoning_content") + if not isinstance(reasoning_content, list): + continue + thinking_blocks: list[dict[str, object]] = [ # mutable-ok: consumed immediately below; never escapes as mutable + { + "type": "thinking", + "thinking": item.get("content") or "", # pyright: ignore[reportUnknownMemberType] # isinstance guard in comprehension filter; pyright can't narrow + "signature": item.get( # pyright: ignore[reportUnknownMemberType] # isinstance guard in comprehension filter; pyright can't narrow + "signature" + ), + } + for item in reasoning_content # pyright: ignore[reportUnknownVariableType] # isinstance(reasoning_content, list) guard above; pyright can't narrow into comprehension + if isinstance(item, dict) + ] + carrier["thinking_blocks"] = thinking_blocks + carrier["reasoning_content"] = ( + "\n".join(str(block["thinking"]) for block in thinking_blocks if block["thinking"]) or None + ) + return choice + + +def normalize_reasoning_content( + raw: dict[str, object], # mutable-ok: wire dicts from httpx JSON; callers own them +) -> dict[str, object]: # mutable-ok: wire dicts from httpx JSON; callers own them + """Transform reasoning_content from SAP Orchestration format [{"content": ..., "signature": ...}] to Optional[str] and a thinking_blocks list.""" + choices: Final[list[object]] = list( # mutable-ok: wire list from httpx JSON + raw.get("choices") or [] # pyright: ignore[reportArgumentType, reportUnknownArgumentType] # wire JSON; choices is always a list + ) + normalized_choices: Final[list[dict[str, object]]] = [ # mutable-ok: wire list passed to pydantic model_validate + normalize_choice(choice) # pyright: ignore[reportUnknownArgumentType] # isinstance guard in comprehension filter; pyright can't narrow + for choice in choices + if isinstance(choice, dict) + ] + normalized_result: Final[dict[str, object]] = { # mutable-ok: wire dict consumed by pydantic model_validate + **raw, + "choices": normalized_choices, + } + return normalized_result + + +def denormalize_reasoning_content( + message: dict[str, object], # mutable-ok: wire dicts from httpx JSON; callers own them +) -> dict[str, object]: # mutable-ok: wire dicts from httpx JSON; callers own them + """Inverse of normalize_reasoning_content: rebuild list[ReasoningBlock] from thinking_blocks. + + LiteLLM normalizes SAP's reasoning_content list into a str + thinking_blocks before + returning a ModelResponse. When that response is replayed as a history message the dict + must be converted back to the SAP shape before SAPAssistantMessage validation. + + If thinking_blocks is absent but reasoning_content is a plain str (e.g. from a + non-SAP provider), drop the field: we cannot reconstruct SAP ReasoningBlocks without + the original signatures. + """ + if message.get("role") != "assistant": + return message + thinking_blocks: Final = message.get("thinking_blocks") + if isinstance(thinking_blocks, list): + reasoning_content: Final[list[dict[str, str | None]]] = [ # mutable-ok: wire list passed to validate_dict + { + "content": str( + block.get("thinking") or "" # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # isinstance guard in comprehension filter; pyright can't narrow + ), + "signature": block.get( # pyright: ignore[reportUnknownMemberType] # isinstance guard in comprehension filter; pyright can't narrow + "signature" + ), + } + for block in thinking_blocks # pyright: ignore[reportUnknownVariableType] # isinstance(thinking_blocks, list) guard above; pyright can't narrow into comprehension + if isinstance(block, dict) + ] + return {k: v for k, v in message.items() if k not in ("thinking_blocks", "reasoning_content")} | { + "reasoning_content": reasoning_content + } + if isinstance(message.get("reasoning_content"), str): + return {k: v for k, v in message.items() if k not in ("thinking_blocks", "reasoning_content")} + return message + + def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool: """OpenAI-shaped chunk is terminal if any choice has a non-None finish_reason.""" try: @@ -47,33 +132,37 @@ class _StreamParser: """Normalize orchestration streaming events into OpenAI-like chunks.""" @staticmethod - def _from_orchestration_result(evt: dict) -> OpenAIChatCompletionChunk | None: + def _from_orchestration_result(evt: dict[str, object]) -> OpenAIChatCompletionChunk | None: """ Accepts orchestration_result shape and maps it to an OpenAI-like *chunk*. """ - orc: Final = evt.get("orchestration_result") or {} + orc: Final[dict[str, object]] = evt.get("orchestration_result") or {} # pyright: ignore[reportAssignmentType] # wire JSON; orchestration_result is always a dict if not orc: return None return OpenAIChatCompletionChunk.model_validate( - { - "id": orc.get("id") or evt.get("request_id") or "stream-chunk", - "object": orc.get("object") or "chat.completion.chunk", - "created": orc.get("created") or evt.get("created") or _now_ts(), - "model": orc.get("model") or "unknown", - "choices": [ - { - "index": c.get("index", 0), - "delta": c.get("delta") or {}, - "finish_reason": c.get("finish_reason"), - } - for c in (orc.get("choices") or []) - ], - } + normalize_reasoning_content( + { + "id": orc.get("id") + or evt.get("request_id") # pyright: ignore[reportUnknownMemberType] # wire JSON string field + or "stream-chunk", + "object": orc.get("object") or "chat.completion.chunk", + "created": orc.get("created") or evt.get("created") or _now_ts(), + "model": orc.get("model") or "unknown", + "choices": [ + { + "index": choice.get("index", 0), + "delta": choice.get("delta") or {}, + "finish_reason": choice.get("finish_reason"), + } + for choice in orc.get("choices") or [] # pyright: ignore[reportUnknownVariableType, reportGeneralTypeIssues] # wire JSON; choices is always list[dict] + ], + } + ) ) @staticmethod - def to_openai_chunk(event_obj: dict) -> OpenAIChatCompletionChunk | None: + def to_openai_chunk(event_obj: dict[str, object]) -> OpenAIChatCompletionChunk | None: """ Accepts: - {"final_result": } (IMPORTANT: this is just another chunk, NOT terminal) @@ -89,11 +178,14 @@ class _StreamParser: # FINAL RESULT IS *NOT* TERMINAL: treat it as the next chunk if "final_result" in event_obj: - fr: Final = event_obj["final_result"] or {} - # ensure it looks like an OpenAI chunk - if "object" not in fr: - fr["object"] = "chat.completion.chunk" - return OpenAIChatCompletionChunk.model_validate(fr) + fr_raw: Final = event_obj["final_result"] or {} + if not isinstance(fr_raw, dict): + return None + fr: Final[dict[str, object]] = { # mutable-ok: wire dict consumed by model_validate + **fr_raw, + **(({"object": "chat.completion.chunk"}) if "object" not in fr_raw else {}), + } + return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(fr)) # Orchestration incremental delta if "orchestration_result" in event_obj: @@ -101,7 +193,7 @@ class _StreamParser: # Already an OpenAI-like chunk if "choices" in event_obj and "object" in event_obj: - return OpenAIChatCompletionChunk.model_validate(event_obj) + return OpenAIChatCompletionChunk.model_validate(normalize_reasoning_content(event_obj)) # Unknown / heartbeat / metrics return None @@ -248,7 +340,8 @@ class AsyncSAPStreamIterator: # ------------------------------- class GenAIHubOrchestration(BaseLLMHTTPHandler): def _add_stream_param_to_request_body(self, data: dict, provider_config: BaseConfig, fake_stream: bool): - if data.get("config", {}).get("stream", None) is not None: + config = data.get("config") or {} # pyright: ignore[reportUnknownVariableType] # wire JSON; config is always a dict when present + if isinstance(config, dict) and config.get("stream") is not None: data["config"]["stream"]["enabled"] = True else: data["config"]["stream"] = {"enabled": True} diff --git a/litellm/llms/sap/chat/models.py b/litellm/llms/sap/chat/models.py index f2b31d4f9b0..a1954e86b2a 100644 --- a/litellm/llms/sap/chat/models.py +++ b/litellm/llms/sap/chat/models.py @@ -133,11 +133,21 @@ class SAPUserMessage(LiteLLMBaseModel): content: str | TextContent | ImageContent | list[TextContent | ImageContent] +class ReasoningBlock( # frozen-ok: adding model_config would override LiteLLMBaseModel's defer_build setting + LiteLLMBaseModel +): + content: str | None = None + signature: str | None = None + + class SAPAssistantMessage(LiteLLMBaseModel): role: Literal["assistant"] = "assistant" content: str | TextContent | list[TextContent] = "" refusal: str = "" - tool_calls: list[MessageToolCall] = Field(default=[]) + tool_calls: list[MessageToolCall] = Field( # mutable-ok: pydantic field; callers append tool calls to this list + default=[] + ) + reasoning_content: list[ReasoningBlock] | None = None # mutable-ok: list of reasoning blocks from SAP wire format class SAPToolChatMessage(LiteLLMBaseModel): diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 0270f814328..3eab22af08f 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -31,6 +31,8 @@ from .handler import ( AsyncSAPStreamIterator, GenAIHubOrchestrationError, SAPStreamIterator, + denormalize_reasoning_content, + normalize_reasoning_content, ) from .models import ( ChatCompletionTool, @@ -106,8 +108,9 @@ def _messages_to_sap_template(messages: list[AllMessageValues]) -> list: template: Final = [] for message in messages: folded_message = _fold_message_cache_control(message_with_file_parts_as_image_urls(message)) - message_role_class = _message_role_mapping(str(folded_message["role"])) - validated_message = validate_dict(folded_message, message_role_class) + denormalized_message = denormalize_reasoning_content(dict(folded_message)) + message_role_class = _message_role_mapping(str(denormalized_message["role"])) + validated_message = validate_dict(denormalized_message, message_role_class) template.append(validated_message) return template @@ -229,7 +232,9 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): "functions", "extra_headers", "parallel_tool_calls", + "reasoning_effort", "response_format", + "thinking", "timeout", ] # Remove response_format for providers that don't support it on SAP GenAI Hub @@ -410,7 +415,9 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): original_response=raw_response.text, additional_args={"complete_input_dict": request_data}, ) - response = ModelResponse.model_validate(raw_response.json()["final_result"]) + response: Final[ModelResponse] = ModelResponse.model_validate( + normalize_reasoning_content(raw_response.json()["final_result"]) + ) # Strip markdown code blocks if JSON response_format was used with Anthropic models # SAP GenAI Hub with Anthropic models sometimes wraps JSON in ```json ... ``` @@ -419,7 +426,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): response_format: Final = optional_params.get("response_format", {}) if response_format.get("type") in ("json_object", "json_schema"): if model.startswith("anthropic"): - response = self._strip_markdown_json(response) + return self._strip_markdown_json(response) return response diff --git a/tests/unit/llms/sap/chat/test_sap_transformation.py b/tests/unit/llms/sap/chat/test_sap_transformation.py index f95bf9b7845..457419e1074 100644 --- a/tests/unit/llms/sap/chat/test_sap_transformation.py +++ b/tests/unit/llms/sap/chat/test_sap_transformation.py @@ -941,3 +941,341 @@ class TestCacheControl: content = self._template(body)[0]["content"] assert "cache_control" not in content[0] assert content[1].get("cache_control") == {"type": "ephemeral"} + +class TestNormalizeReasoningContent: + """Unit tests for normalize_reasoning_content.""" + + from litellm.llms.sap.chat.handler import normalize_reasoning_content + + _normalize_reasoning_content = staticmethod(normalize_reasoning_content) + + def test_list_reasoning_content_mapped_to_thinking_blocks(self): + """List-shaped reasoning_content is converted to thinking_blocks.""" + raw = { + "choices": [{ + "message": { + "role": "assistant", + "content": "Latin.", + "reasoning_content": [ + {"content": "Romans spoke Latin.", "signature": "sig1"}, + {"content": "That is well known.", "signature": "sig2"}, + ], + } + }] + } + out = self._normalize_reasoning_content(raw) + msg = out["choices"][0]["message"] + assert msg["thinking_blocks"] == [ + {"type": "thinking", "thinking": "Romans spoke Latin.", "signature": "sig1"}, + {"type": "thinking", "thinking": "That is well known.", "signature": "sig2"}, + ] + assert msg["reasoning_content"] == "Romans spoke Latin.\nThat is well known." + + def test_string_reasoning_content_unchanged(self): + """String reasoning_content is left as-is (already the right type).""" + raw = { + "choices": [{ + "message": { + "role": "assistant", + "content": "42", + "reasoning_content": "I thought about it.", + } + }] + } + out = self._normalize_reasoning_content(raw) + msg = out["choices"][0]["message"] + assert msg["reasoning_content"] == "I thought about it." + assert "thinking_blocks" not in msg + + def test_no_reasoning_content_unchanged(self): + """A message without reasoning_content is not modified.""" + raw = {"choices": [{"message": {"role": "assistant", "content": "Hi."}}]} + out = self._normalize_reasoning_content(raw) + assert out == raw + + def test_empty_list_reasoning_content_sets_none(self): + """An empty list produces None for reasoning_content and empty thinking_blocks.""" + raw = {"choices": [{"message": {"reasoning_content": []}}]} + out = self._normalize_reasoning_content(raw) + msg = out["choices"][0]["message"] + assert msg["thinking_blocks"] == [] + assert msg["reasoning_content"] is None + + def test_multiple_choices_all_normalized(self): + """All choices in the response are normalized.""" + raw = { + "choices": [ + {"message": {"reasoning_content": [{"content": "thought A", "signature": None}]}}, + {"message": {"reasoning_content": [{"content": "thought B", "signature": "s"}]}}, + ] + } + out = self._normalize_reasoning_content(raw) + assert out["choices"][0]["message"]["reasoning_content"] == "thought A" + assert out["choices"][1]["message"]["reasoning_content"] == "thought B" + + + def test_null_content_in_block_uses_empty_string(self): + """Explicit null content value must not leak None into thinking field.""" + raw = { + "choices": [{ + "message": { + "reasoning_content": [{"content": None, "signature": "s"}], + } + }] + } + out = self._normalize_reasoning_content(raw) + block = out["choices"][0]["message"]["thinking_blocks"][0] + assert block["thinking"] == "" + assert out["choices"][0]["message"]["reasoning_content"] is None + + def test_transform_response_normalizes_list_reasoning_content(self): + """Production path: transform_response must produce a ModelResponse + with thinking_blocks populated when the raw payload carries a + list-shaped reasoning_content. + """ + import json + from unittest.mock import MagicMock + + from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig + + config = GenAIHubOrchestrationConfig() + + final_result = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1700000000, + "model": "gemini-test", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "The answer is 42.", + "reasoning_content": [ + {"content": "Let me think.", "signature": "sig1"}, + {"content": "Yes, 42.", "signature": "sig2"}, + ], + }, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + + raw_response = MagicMock() + raw_response.text = json.dumps({"final_result": final_result}) + raw_response.json.return_value = {"final_result": final_result} + + response = config.transform_response( + model="gemini-test", + raw_response=raw_response, + model_response=MagicMock(), + logging_obj=MagicMock(), + api_key="test", + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + choice = response.choices[0] + assert hasattr(choice.message, "thinking_blocks"), "thinking_blocks missing from message" + assert choice.message.thinking_blocks == [ + {"type": "thinking", "thinking": "Let me think.", "signature": "sig1"}, + {"type": "thinking", "thinking": "Yes, 42.", "signature": "sig2"}, + ] + assert choice.message.reasoning_content == "Let me think.\nYes, 42." + + +class TestNormalizeChoice: + """Unit tests for normalize_choice (message and delta shapes).""" + + from litellm.llms.sap.chat.handler import normalize_choice, normalize_reasoning_content + + _normalize_choice = staticmethod(normalize_choice) + _normalize_reasoning_content = staticmethod(normalize_reasoning_content) + + def test_message_list_reasoning_content_normalized(self): + choice = { + "index": 0, + "message": {"role": "assistant", "content": "hi", "reasoning_content": [{"content": "thought", "signature": "s1"}]}, + "finish_reason": "stop", + } + result = self._normalize_choice(choice) + msg = result["message"] + assert msg["reasoning_content"] == "thought" + assert msg["thinking_blocks"] == [{"type": "thinking", "thinking": "thought", "signature": "s1"}] + + def test_delta_list_reasoning_content_normalized(self): + choice = { + "index": 0, + "delta": {"role": "assistant", "reasoning_content": [{"content": "delta thought", "signature": None}]}, + "finish_reason": None, + } + result = self._normalize_choice(choice) + delta = result["delta"] + assert delta["reasoning_content"] == "delta thought" + assert delta["thinking_blocks"][0]["thinking"] == "delta thought" + + def test_string_reasoning_content_unchanged(self): + choice = {"index": 0, "message": {"reasoning_content": "already a string"}} + assert self._normalize_choice(choice) == choice + + def test_no_reasoning_content_unchanged(self): + choice = {"index": 0, "delta": {"content": "hello"}} + assert self._normalize_choice(choice) == choice + + def test_normalize_reasoning_content_covers_delta_path(self): + raw = { + "id": "c1", + "object": "chat.completion.chunk", + "choices": [ + {"index": 0, "delta": {"reasoning_content": [{"content": "stream thought", "signature": "sig"}]}, "finish_reason": None} + ], + } + result = self._normalize_reasoning_content(raw) + delta = result["choices"][0]["delta"] + assert delta["reasoning_content"] == "stream thought" + assert delta["thinking_blocks"][0]["type"] == "thinking" + + +class TestMessagesToSapTemplateReasoningContent: + """_messages_to_sap_template must forward reasoning_content on assistant turns.""" + + def test_assistant_message_with_reasoning_content_is_preserved(self): + from litellm.llms.sap.chat.transformation import _messages_to_sap_template + + messages = [ + {"role": "user", "content": "What is 22?"}, + { + "role": "assistant", + "content": "4", + "reasoning_content": [ + {"content": "Simple arithmetic.", "signature": "sig1"} + ], + }, + {"role": "user", "content": "Are you sure?"}, + ] + result = _messages_to_sap_template(messages) + assistant_msg = result[1] + assert assistant_msg["reasoning_content"] == [ + {"content": "Simple arithmetic.", "signature": "sig1"} + ] + + def test_litellm_normalized_assistant_message_round_trips_signature(self): + """After a non-streaming SAP call, the ModelResponse has reasoning_content: str + and thinking_blocks: list (LiteLLM native format). Replaying it as a history + message must not raise and must preserve the signature. + """ + from litellm.llms.sap.chat.transformation import _messages_to_sap_template + + signature = "EtwHCkABGAIqQKNbQn0Uz==" + messages = [ + {"role": "user", "content": "Translate it."}, + { + "role": "assistant", + "content": "The text is Lorem Ipsum.", + "reasoning_content": "Lorem ipsum is pseudo-Latin...", + "thinking_blocks": [ + {"type": "thinking", "thinking": "Lorem ipsum is pseudo-Latin...", "signature": signature} + ], + }, + {"role": "user", "content": "Summarise it."}, + ] + + result = _messages_to_sap_template(messages) + assistant = next(m for m in result if m.get("role") == "assistant") + + reasoning = assistant.get("reasoning_content") + assert isinstance(reasoning, list) + assert any(block.get("signature") == signature for block in reasoning) + + def test_streaming_assembled_assistant_message_round_trips_signature(self): + """After a streaming SAP call, normalize_reasoning_content runs on every chunk. + The assembled assistant message has the same LiteLLM-native shape as the + non-streaming case. Verify the signature survives the round-trip. + """ + from litellm.llms.sap.chat.handler import normalize_reasoning_content + from litellm.llms.sap.chat.transformation import _messages_to_sap_template + + signature = "EtwHCkABGAIqQKNbQn0Uz==" + thinking_text = "Lorem ipsum is pseudo-Latin..." + sap_final_result = { + "id": "msg_stream_01", + "object": "chat.completion", + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The text is Lorem Ipsum.", + "reasoning_content": [{"content": thinking_text, "signature": signature}], + }, + "finish_reason": "stop", + } + ], + } + normalized = normalize_reasoning_content(sap_final_result) + assembled_message = {"role": "assistant", **normalized["choices"][0]["message"]} + + messages = [ + {"role": "user", "content": "Translate it."}, + assembled_message, + {"role": "user", "content": "Summarize it."}, + ] + + result = _messages_to_sap_template(messages) + assistant = next(m for m in result if m.get("role") == "assistant") + + reasoning = assistant.get("reasoning_content") + assert isinstance(reasoning, list) + assert any(block.get("signature") == signature for block in reasoning) + + +class TestDenormalizeReasoningContent: + from litellm.llms.sap.chat.handler import denormalize_reasoning_content + + _denormalize = staticmethod(denormalize_reasoning_content) + + def test_thinking_blocks_rebuilt_into_sap_reasoning_content(self): + message = { + "role": "assistant", + "content": "hello", + "thinking_blocks": [{"type": "thinking", "thinking": "step one", "signature": "sig1"}], + "reasoning_content": "step one", + } + result = self._denormalize(message) + assert result["reasoning_content"] == [{"content": "step one", "signature": "sig1"}] + assert "thinking_blocks" not in result + + def test_string_reasoning_content_without_thinking_blocks_is_dropped(self): + message = { + "role": "assistant", + "content": "hello", + "reasoning_content": "some thought from another provider", + } + result = self._denormalize(message) + assert "reasoning_content" not in result + assert "thinking_blocks" not in result + assert result["content"] == "hello" + + def test_non_assistant_message_is_returned_unchanged(self): + message = {"role": "user", "content": "hi", "reasoning_content": "should stay"} + assert self._denormalize(message) is message + + def test_assistant_without_reasoning_fields_is_returned_unchanged(self): + message = {"role": "assistant", "content": "hi"} + assert self._denormalize(message) is message + + def test_validate_dict_accepts_denormalized_message_from_string_reasoning_content(self): + from litellm.llms.sap.chat.models import SAPAssistantMessage + from litellm.llms.sap.chat.transformation import validate_dict + + message = { + "role": "assistant", + "content": "done", + "reasoning_content": "raw string from openai o-series", + } + result = validate_dict(self._denormalize(message), SAPAssistantMessage) + assert result["role"] == "assistant" + assert "reasoning_content" not in result or result.get("reasoning_content") is None