From 9538e216b2af31f6f92249ef45cada1f6571eec7 Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Tue, 28 Jul 2026 16:13:43 +0800 Subject: [PATCH 01/11] fix(sap): normalize stream chunks to avoid MockValSer errors and clarify missing deployment error --- litellm/llms/sap/chat/handler.py | 26 ++++++++++++++++++++++--- litellm/llms/sap/chat/transformation.py | 9 +++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index b4f4d4faddb..77fb59431e9 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -9,6 +9,7 @@ import httpx from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.openai import OpenAIChatCompletionChunk +from litellm.types.utils import Usage from ...custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -46,6 +47,25 @@ def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool: class _StreamParser: """Normalize orchestration streaming events into OpenAI-like chunks.""" + @staticmethod + def _validate_chunk(payload: dict) -> OpenAIChatCompletionChunk: + """ + Validate an OpenAI-shaped dict into a chunk, normalizing fields that would + otherwise break downstream serialization: + - drop the empty `logprobs` ({}) the orchestration service sends on every choice + - replace the raw openai-SDK usage object (deferred-build pydantic model whose + serializer is still a MockValSer) with litellm's Usage, so nested + model_dump() calls in the streaming handler don't raise + "'MockValSer' object is not an instance of 'SchemaSerializer'" + """ + for choice in payload.get("choices") or []: + if isinstance(choice, dict) and not choice.get("logprobs"): + choice.pop("logprobs", None) + chunk = OpenAIChatCompletionChunk.model_validate(payload) + if chunk.usage is not None: + chunk.usage = Usage(**chunk.usage.model_dump()) + return chunk + @staticmethod def _from_orchestration_result(evt: dict) -> OpenAIChatCompletionChunk | None: """ @@ -55,7 +75,7 @@ class _StreamParser: if not orc: return None - return OpenAIChatCompletionChunk.model_validate( + return _StreamParser._validate_chunk( { "id": orc.get("id") or evt.get("request_id") or "stream-chunk", "object": orc.get("object") or "chat.completion.chunk", @@ -93,7 +113,7 @@ class _StreamParser: # ensure it looks like an OpenAI chunk if "object" not in fr: fr["object"] = "chat.completion.chunk" - return OpenAIChatCompletionChunk.model_validate(fr) + return _StreamParser._validate_chunk(fr) # Orchestration incremental delta if "orchestration_result" in event_obj: @@ -101,7 +121,7 @@ class _StreamParser: # Already an OpenAI-like chunk if "choices" in event_obj and "object" in event_obj: - return OpenAIChatCompletionChunk.model_validate(event_obj) + return _StreamParser._validate_chunk(event_obj) # Unknown / heartbeat / metrics return None diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index d64d7a57281..affb6193287 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -183,6 +183,15 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): if cfg.get("executableId") == "orchestration": valid.append((dep["deploymentUrl"], dep["createdAt"])) # newest first + if not valid: + raise GenAIHubOrchestrationError( + status_code=404, + message=( + "No orchestration deployment found in SAP AI Core resource group " + f"'{self.resource_group}'. Create/start an orchestration deployment " + "in SAP AI Launchpad, then retry." + ), + ) return sorted(valid, key=lambda x: x[1], reverse=True)[0][0] @classmethod From 045e21b7a6331c837d515df4f58469c8cb0d4287 Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Tue, 28 Jul 2026 16:30:49 +0800 Subject: [PATCH 02/11] test(sap): add unit tests for stream chunk normalization and missing deployment error --- .../chat/test_sap_stream_chunk_validation.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py diff --git a/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py b/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py new file mode 100644 index 00000000000..018cd89e195 --- /dev/null +++ b/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py @@ -0,0 +1,104 @@ +""" +Tests for SAP orchestration stream chunk normalization (_StreamParser._validate_chunk) +and the descriptive error raised when no orchestration deployment exists. + +Regression tests for: +- TypeError: 'MockValSer' object is not an instance of 'SchemaSerializer' + raised from nested model_dump() when the raw openai-SDK usage object + (a deferred-build pydantic model) was attached to ModelResponseStream. +- IndexError: list index out of range raised from deployment_url when the + configured resource group contains no orchestration deployment. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError, _StreamParser +from litellm.types.utils import ModelResponseStream, Usage + + +def _final_chunk_payload() -> dict: + """OpenAI-shaped final chunk as sent by the SAP orchestration service: + every choice carries an empty `logprobs` ({}) and the last chunk carries usage.""" + return { + "id": "chatcmpl-sap-final", + "object": "chat.completion.chunk", + "created": 1761319270, + "model": "anthropic--claude-4.7-opus", + "choices": [ + { + "index": 0, + "delta": {}, + "logprobs": {}, + "finish_reason": "tool_calls", + } + ], + "usage": { + "completion_tokens": 206, + "prompt_tokens": 62322, + "total_tokens": 62528, + }, + } + + +def test_validate_chunk_drops_empty_logprobs(): + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + assert chunk.choices[0].logprobs is None + + +def test_validate_chunk_preserves_real_logprobs(): + payload = _final_chunk_payload() + payload["choices"][0]["logprobs"] = { + "content": [{"token": "Hello", "logprob": -0.1, "bytes": None, "top_logprobs": []}] + } + chunk = _StreamParser._validate_chunk(payload) + assert chunk.choices[0].logprobs is not None + assert chunk.choices[0].logprobs.content[0].token == "Hello" + + +def test_validate_chunk_converts_usage_to_litellm_usage(): + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + assert isinstance(chunk.usage, Usage) + assert chunk.usage.completion_tokens == 206 + assert chunk.usage.prompt_tokens == 62322 + assert chunk.usage.total_tokens == 62528 + + +def test_validate_chunk_without_usage_keeps_none(): + payload = _final_chunk_payload() + del payload["usage"] + chunk = _StreamParser._validate_chunk(payload) + assert chunk.usage is None + + +def test_validated_usage_survives_nested_model_dump(): + """The original crash: the raw openai-SDK usage object attached to a + ModelResponseStream made model_dump() raise + "TypeError: 'MockValSer' object is not an instance of 'SchemaSerializer'".""" + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + + model_response = ModelResponseStream() + setattr(model_response, "usage", chunk.usage) + + dumped = model_response.model_dump() # must not raise + assert dumped["usage"]["total_tokens"] == 62528 + + +def test_deployment_url_raises_404_when_no_orchestration_deployment(): + from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig + + config = GenAIHubOrchestrationConfig() + config.token_creator = lambda: "Bearer FAKE_TOKEN" + config._base_url = "https://api.ai.mock-sap.com/v2" + config._resource_group = "fake-group" + + mock_client = MagicMock() + mock_client.get.return_value.json.return_value = {"resources": []} + + with patch("litellm.module_level_client", mock_client): + with pytest.raises(GenAIHubOrchestrationError) as exc_info: + _ = config.deployment_url + + assert exc_info.value.status_code == 404 + assert "fake-group" in exc_info.value.message From 400f686e06ff1c14d1fe3da9055b5497b4a7a393 Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Tue, 28 Jul 2026 18:35:19 +0800 Subject: [PATCH 03/11] chore(sap): suppress LIT001 on _validate_chunk payload param with reason --- litellm/llms/sap/chat/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index 77fb59431e9..91cf53dfab7 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -48,7 +48,7 @@ class _StreamParser: """Normalize orchestration streaming events into OpenAI-like chunks.""" @staticmethod - def _validate_chunk(payload: dict) -> OpenAIChatCompletionChunk: + def _validate_chunk(payload: dict) -> OpenAIChatCompletionChunk: # mutable-ok: normalizes the raw SSE dict in place (pops empty logprobs) before validation """ Validate an OpenAI-shaped dict into a chunk, normalizing fields that would otherwise break downstream serialization: From 216455248e56f08eb1bb7a7b81101cd773a86298 Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Tue, 28 Jul 2026 18:44:07 +0800 Subject: [PATCH 04/11] test(sap): cover to_openai_chunk normalization branches --- .../chat/test_sap_stream_chunk_validation.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py b/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py index 018cd89e195..b2dc73cf846 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py @@ -85,6 +85,39 @@ def test_validated_usage_survives_nested_model_dump(): assert dumped["usage"]["total_tokens"] == 62528 +def test_to_openai_chunk_normalizes_openai_shaped_event(): + """An already-openai-shaped event goes through the same normalization.""" + chunk = _StreamParser.to_openai_chunk(_final_chunk_payload()) + assert chunk is not None + assert chunk.choices[0].logprobs is None + assert isinstance(chunk.usage, Usage) + + +def test_to_openai_chunk_from_orchestration_result(): + """An orchestration_result delta event is mapped and normalized into a chunk.""" + event = { + "request_id": "a07127d3-cb74-9427-a4dc-ef9bf424fb43", + "orchestration_result": { + "id": "chatcmpl-sap-delta", + "object": "chat.completion.chunk", + "created": 1761319270, + "model": "anthropic--claude-4.7-opus", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hello "}, + "logprobs": {}, + "finish_reason": None, + } + ], + }, + } + chunk = _StreamParser.to_openai_chunk(event) + assert chunk is not None + assert chunk.choices[0].delta.content == "Hello " + assert chunk.choices[0].logprobs is None + + def test_deployment_url_raises_404_when_no_orchestration_deployment(): from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig From 27e45cd855c833d26300132c1f2dd1b0d6ae516e Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Tue, 28 Jul 2026 18:54:34 +0800 Subject: [PATCH 05/11] style(sap): satisfy ruff format while keeping LIT001 suppression on param line --- litellm/llms/sap/chat/handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index 91cf53dfab7..db8973815b7 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -48,7 +48,9 @@ class _StreamParser: """Normalize orchestration streaming events into OpenAI-like chunks.""" @staticmethod - def _validate_chunk(payload: dict) -> OpenAIChatCompletionChunk: # mutable-ok: normalizes the raw SSE dict in place (pops empty logprobs) before validation + def _validate_chunk( + payload: dict, # mutable-ok: normalized in place (pops empty logprobs) before validation + ) -> OpenAIChatCompletionChunk: """ Validate an OpenAI-shaped dict into a chunk, normalizing fields that would otherwise break downstream serialization: From 70e540ddc86412cd5058e3ca410ddfadaf62f8d3 Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Tue, 4 Aug 2026 10:50:58 +0800 Subject: [PATCH 06/11] chore(sap): suppress LIT002 on choices fallback list with reason --- litellm/llms/sap/chat/handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index db8973815b7..53537726047 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -60,7 +60,7 @@ class _StreamParser: model_dump() calls in the streaming handler don't raise "'MockValSer' object is not an instance of 'SchemaSerializer'" """ - for choice in payload.get("choices") or []: + for choice in payload.get("choices") or []: # mutable-ok: only iterated, never mutated if isinstance(choice, dict) and not choice.get("logprobs"): choice.pop("logprobs", None) chunk = OpenAIChatCompletionChunk.model_validate(payload) From 77890478a1fd6b0ea433177163f027b26880d21f Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Mon, 7 Sep 2026 13:58:16 +0800 Subject: [PATCH 07/11] refactor(sap): inject http client into deployment_url to drop internal patch Replace the test's patch of litellm.module_level_client with dependency injection via a new optional _http_client attribute, clearing the TQ008 test-quality violation. Production still falls back to the module-level client when none is injected. --- litellm/llms/sap/chat/transformation.py | 6 +++--- .../llms/sap/chat/test_sap_stream_chunk_validation.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index affb6193287..f41a4c3f336 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: import tiktoken from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -136,6 +137,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): self.token_creator = None self._base_url = None self._resource_group = None + self._http_client: HTTPHandler | None = None def run_env_setup(self, service_key: str | None = None) -> None: try: @@ -169,9 +171,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): @cached_property def deployment_url(self) -> str: - # Keep a short, tight client lifecycle here to avoid fd leaks - client: Final = litellm.module_level_client - # with httpx.Client(timeout=30) as client: + client: Final = self._http_client if self._http_client is not None else litellm.module_level_client deployments: Final = client.get(f"{self.base_url}/lm/deployments", headers=self.headers).json() valid: Final[list[tuple[str, str]]] = [] for dep in deployments.get("resources", []): diff --git a/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py b/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py index b2dc73cf846..632130c23ea 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py @@ -10,7 +10,7 @@ Regression tests for: configured resource group contains no orchestration deployment. """ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import pytest @@ -128,10 +128,10 @@ def test_deployment_url_raises_404_when_no_orchestration_deployment(): mock_client = MagicMock() mock_client.get.return_value.json.return_value = {"resources": []} + config._http_client = mock_client - with patch("litellm.module_level_client", mock_client): - with pytest.raises(GenAIHubOrchestrationError) as exc_info: - _ = config.deployment_url + with pytest.raises(GenAIHubOrchestrationError) as exc_info: + _ = config.deployment_url assert exc_info.value.status_code == 404 assert "fake-group" in exc_info.value.message From 264d05b1fa0d241a5d0eaf4bde9822483c07b13b Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Mon, 7 Sep 2026 14:20:16 +0800 Subject: [PATCH 08/11] test(sap): consolidate regression tests and drop explanatory comments Address Greptile review: move the stream-chunk normalization cases into test_sap_chat_calls.py and the deployment_url 404 case into test_sap_transformation.py, then delete the standalone module so bug-fix regressions extend the mapped SAP test files. Strip the explanatory docstring from _validate_chunk, keeping only the mutable-ok suppressions. --- litellm/llms/sap/chat/handler.py | 9 -- .../llms/sap/chat/test_sap_chat_calls.py | 110 ++++++++++++++ .../chat/test_sap_stream_chunk_validation.py | 137 ------------------ .../llms/sap/chat/test_sap_transformation.py | 15 ++ 4 files changed, 125 insertions(+), 146 deletions(-) delete mode 100644 tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index 53537726047..2867b64057b 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -51,15 +51,6 @@ class _StreamParser: def _validate_chunk( payload: dict, # mutable-ok: normalized in place (pops empty logprobs) before validation ) -> OpenAIChatCompletionChunk: - """ - Validate an OpenAI-shaped dict into a chunk, normalizing fields that would - otherwise break downstream serialization: - - drop the empty `logprobs` ({}) the orchestration service sends on every choice - - replace the raw openai-SDK usage object (deferred-build pydantic model whose - serializer is still a MockValSer) with litellm's Usage, so nested - model_dump() calls in the streaming handler don't raise - "'MockValSer' object is not an instance of 'SchemaSerializer'" - """ for choice in payload.get("choices") or []: # mutable-ok: only iterated, never mutated if isinstance(choice, dict) and not choice.get("logprobs"): choice.pop("logprobs", None) diff --git a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py b/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py index c41254f1a2b..170354dae7e 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py @@ -204,3 +204,113 @@ async def test_sap_chat_required_headers( f"Header '{header_name}' has incorrect value. " f"Expected: '{expected_value}', Got: '{request.headers[header_name]}'" ) + + +def _final_chunk_payload() -> dict: + return { + "id": "chatcmpl-sap-final", + "object": "chat.completion.chunk", + "created": 1761319270, + "model": "anthropic--claude-4.7-opus", + "choices": [ + { + "index": 0, + "delta": {}, + "logprobs": {}, + "finish_reason": "tool_calls", + } + ], + "usage": { + "completion_tokens": 206, + "prompt_tokens": 62322, + "total_tokens": 62528, + }, + } + + +def test_validate_chunk_drops_empty_logprobs(): + from litellm.llms.sap.chat.handler import _StreamParser + + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + assert chunk.choices[0].logprobs is None + + +def test_validate_chunk_preserves_real_logprobs(): + from litellm.llms.sap.chat.handler import _StreamParser + + payload = _final_chunk_payload() + payload["choices"][0]["logprobs"] = { + "content": [{"token": "Hello", "logprob": -0.1, "bytes": None, "top_logprobs": []}] + } + chunk = _StreamParser._validate_chunk(payload) + assert chunk.choices[0].logprobs is not None + assert chunk.choices[0].logprobs.content[0].token == "Hello" + + +def test_validate_chunk_converts_usage_to_litellm_usage(): + from litellm.llms.sap.chat.handler import _StreamParser + from litellm.types.utils import Usage + + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + assert isinstance(chunk.usage, Usage) + assert chunk.usage.completion_tokens == 206 + assert chunk.usage.prompt_tokens == 62322 + assert chunk.usage.total_tokens == 62528 + + +def test_validate_chunk_without_usage_keeps_none(): + from litellm.llms.sap.chat.handler import _StreamParser + + payload = _final_chunk_payload() + del payload["usage"] + chunk = _StreamParser._validate_chunk(payload) + assert chunk.usage is None + + +def test_validated_usage_survives_nested_model_dump(): + from litellm.llms.sap.chat.handler import _StreamParser + from litellm.types.utils import ModelResponseStream + + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + + model_response = ModelResponseStream() + setattr(model_response, "usage", chunk.usage) + + dumped = model_response.model_dump() + assert dumped["usage"]["total_tokens"] == 62528 + + +def test_to_openai_chunk_normalizes_openai_shaped_event(): + from litellm.llms.sap.chat.handler import _StreamParser + from litellm.types.utils import Usage + + chunk = _StreamParser.to_openai_chunk(_final_chunk_payload()) + assert chunk is not None + assert chunk.choices[0].logprobs is None + assert isinstance(chunk.usage, Usage) + + +def test_to_openai_chunk_from_orchestration_result(): + from litellm.llms.sap.chat.handler import _StreamParser + + event = { + "request_id": "a07127d3-cb74-9427-a4dc-ef9bf424fb43", + "orchestration_result": { + "id": "chatcmpl-sap-delta", + "object": "chat.completion.chunk", + "created": 1761319270, + "model": "anthropic--claude-4.7-opus", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hello "}, + "logprobs": {}, + "finish_reason": None, + } + ], + }, + } + chunk = _StreamParser.to_openai_chunk(event) + assert chunk is not None + assert chunk.choices[0].delta.content == "Hello " + assert chunk.choices[0].logprobs is None diff --git a/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py b/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py deleted file mode 100644 index 632130c23ea..00000000000 --- a/tests/test_litellm/llms/sap/chat/test_sap_stream_chunk_validation.py +++ /dev/null @@ -1,137 +0,0 @@ -""" -Tests for SAP orchestration stream chunk normalization (_StreamParser._validate_chunk) -and the descriptive error raised when no orchestration deployment exists. - -Regression tests for: -- TypeError: 'MockValSer' object is not an instance of 'SchemaSerializer' - raised from nested model_dump() when the raw openai-SDK usage object - (a deferred-build pydantic model) was attached to ModelResponseStream. -- IndexError: list index out of range raised from deployment_url when the - configured resource group contains no orchestration deployment. -""" - -from unittest.mock import MagicMock - -import pytest - -from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError, _StreamParser -from litellm.types.utils import ModelResponseStream, Usage - - -def _final_chunk_payload() -> dict: - """OpenAI-shaped final chunk as sent by the SAP orchestration service: - every choice carries an empty `logprobs` ({}) and the last chunk carries usage.""" - return { - "id": "chatcmpl-sap-final", - "object": "chat.completion.chunk", - "created": 1761319270, - "model": "anthropic--claude-4.7-opus", - "choices": [ - { - "index": 0, - "delta": {}, - "logprobs": {}, - "finish_reason": "tool_calls", - } - ], - "usage": { - "completion_tokens": 206, - "prompt_tokens": 62322, - "total_tokens": 62528, - }, - } - - -def test_validate_chunk_drops_empty_logprobs(): - chunk = _StreamParser._validate_chunk(_final_chunk_payload()) - assert chunk.choices[0].logprobs is None - - -def test_validate_chunk_preserves_real_logprobs(): - payload = _final_chunk_payload() - payload["choices"][0]["logprobs"] = { - "content": [{"token": "Hello", "logprob": -0.1, "bytes": None, "top_logprobs": []}] - } - chunk = _StreamParser._validate_chunk(payload) - assert chunk.choices[0].logprobs is not None - assert chunk.choices[0].logprobs.content[0].token == "Hello" - - -def test_validate_chunk_converts_usage_to_litellm_usage(): - chunk = _StreamParser._validate_chunk(_final_chunk_payload()) - assert isinstance(chunk.usage, Usage) - assert chunk.usage.completion_tokens == 206 - assert chunk.usage.prompt_tokens == 62322 - assert chunk.usage.total_tokens == 62528 - - -def test_validate_chunk_without_usage_keeps_none(): - payload = _final_chunk_payload() - del payload["usage"] - chunk = _StreamParser._validate_chunk(payload) - assert chunk.usage is None - - -def test_validated_usage_survives_nested_model_dump(): - """The original crash: the raw openai-SDK usage object attached to a - ModelResponseStream made model_dump() raise - "TypeError: 'MockValSer' object is not an instance of 'SchemaSerializer'".""" - chunk = _StreamParser._validate_chunk(_final_chunk_payload()) - - model_response = ModelResponseStream() - setattr(model_response, "usage", chunk.usage) - - dumped = model_response.model_dump() # must not raise - assert dumped["usage"]["total_tokens"] == 62528 - - -def test_to_openai_chunk_normalizes_openai_shaped_event(): - """An already-openai-shaped event goes through the same normalization.""" - chunk = _StreamParser.to_openai_chunk(_final_chunk_payload()) - assert chunk is not None - assert chunk.choices[0].logprobs is None - assert isinstance(chunk.usage, Usage) - - -def test_to_openai_chunk_from_orchestration_result(): - """An orchestration_result delta event is mapped and normalized into a chunk.""" - event = { - "request_id": "a07127d3-cb74-9427-a4dc-ef9bf424fb43", - "orchestration_result": { - "id": "chatcmpl-sap-delta", - "object": "chat.completion.chunk", - "created": 1761319270, - "model": "anthropic--claude-4.7-opus", - "choices": [ - { - "index": 0, - "delta": {"role": "assistant", "content": "Hello "}, - "logprobs": {}, - "finish_reason": None, - } - ], - }, - } - chunk = _StreamParser.to_openai_chunk(event) - assert chunk is not None - assert chunk.choices[0].delta.content == "Hello " - assert chunk.choices[0].logprobs is None - - -def test_deployment_url_raises_404_when_no_orchestration_deployment(): - from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig - - config = GenAIHubOrchestrationConfig() - config.token_creator = lambda: "Bearer FAKE_TOKEN" - config._base_url = "https://api.ai.mock-sap.com/v2" - config._resource_group = "fake-group" - - mock_client = MagicMock() - mock_client.get.return_value.json.return_value = {"resources": []} - config._http_client = mock_client - - with pytest.raises(GenAIHubOrchestrationError) as exc_info: - _ = config.deployment_url - - assert exc_info.value.status_code == 404 - assert "fake-group" in exc_info.value.message diff --git a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py index 3601bdd0d5e..565545a8d84 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py @@ -639,3 +639,18 @@ class TestSAPTransformationIntegration: config["config"]["modules"][1]["translation"]["input"]["type"] == "sap_document_translation" ) + + def test_deployment_url_raises_404_when_no_orchestration_deployment(self, mock_config): + from unittest.mock import MagicMock + + from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError + + mock_client = MagicMock() + mock_client.get.return_value.json.return_value = {"resources": []} + mock_config._http_client = mock_client + + with pytest.raises(GenAIHubOrchestrationError) as exc_info: + _ = mock_config.deployment_url + + assert exc_info.value.status_code == 404 + assert "test-group" in exc_info.value.message From a215b78d99ea2f853e9bfdf07911f1f94df50ca6 Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Mon, 14 Sep 2026 11:05:04 +0800 Subject: [PATCH 09/11] fix(sap): address review feedback on stream chunk validation Type the chunk payload as dict[str, object] and narrow choices with an isinstance guard before iterating, fix the in-place mutation suppression that claimed the loop never mutated, build the litellm Usage with model_validate instead of spreading a dict[str, Any], and make the usage regression test go through model_dump_json, the path that actually raised MockValSer. model_dump() self-heals via __getattr__, so the old test passed on unfixed code. Carry the same typing into to_openai_chunk and the payload literals it validates, so no basedpyright rule counts higher than before the change. --- litellm/llms/sap/chat/handler.py | 50 ++++++++++--------- .../llms/sap/chat/test_sap_chat_calls.py | 5 +- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index 2867b64057b..d0be0086214 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -49,14 +49,16 @@ class _StreamParser: @staticmethod def _validate_chunk( - payload: dict, # mutable-ok: normalized in place (pops empty logprobs) before validation + payload: dict[str, object], # mutable-ok: normalized in place (pops empty logprobs) before validation ) -> OpenAIChatCompletionChunk: - for choice in payload.get("choices") or []: # mutable-ok: only iterated, never mutated - if isinstance(choice, dict) and not choice.get("logprobs"): - choice.pop("logprobs", None) + choices: Final = payload.get("choices") + if isinstance(choices, list): + for choice in choices: + if isinstance(choice, dict) and not choice.get("logprobs"): + choice.pop("logprobs", None) # mutable-ok: pops the logprobs key in-place before model_validate chunk = OpenAIChatCompletionChunk.model_validate(payload) if chunk.usage is not None: - chunk.usage = Usage(**chunk.usage.model_dump()) + chunk.usage = Usage.model_validate(chunk.usage.model_dump()) return chunk @staticmethod @@ -68,25 +70,24 @@ class _StreamParser: if not orc: return None - return _StreamParser._validate_chunk( - { - "id": orc.get("id") or evt.get("request_id") or "stream-chunk", - "object": orc.get("object") or "chat.completion.chunk", - "created": orc.get("created") or evt.get("created") or _now_ts(), - "model": orc.get("model") or "unknown", - "choices": [ - { - "index": c.get("index", 0), - "delta": c.get("delta") or {}, - "finish_reason": c.get("finish_reason"), - } - for c in (orc.get("choices") or []) - ], - } - ) + payload: Final[dict[str, object]] = { + "id": orc.get("id") or evt.get("request_id") or "stream-chunk", + "object": orc.get("object") or "chat.completion.chunk", + "created": orc.get("created") or evt.get("created") or _now_ts(), + "model": orc.get("model") or "unknown", + "choices": [ + { + "index": c.get("index", 0), + "delta": c.get("delta") or {}, + "finish_reason": c.get("finish_reason"), + } + for c in (orc.get("choices") or []) + ], + } + return _StreamParser._validate_chunk(payload) @staticmethod - def to_openai_chunk(event_obj: dict) -> OpenAIChatCompletionChunk | None: + def to_openai_chunk(event_obj: dict[str, object]) -> OpenAIChatCompletionChunk | None: """ Accepts: - {"final_result": } (IMPORTANT: this is just another chunk, NOT terminal) @@ -102,7 +103,10 @@ class _StreamParser: # FINAL RESULT IS *NOT* TERMINAL: treat it as the next chunk if "final_result" in event_obj: - fr: Final = event_obj["final_result"] or {} + final_result: Final = event_obj["final_result"] + if not isinstance(final_result, dict): + return None + fr: Final[dict[str, object]] = final_result # ensure it looks like an OpenAI chunk if "object" not in fr: fr["object"] = "chat.completion.chunk" diff --git a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py b/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py index 170354dae7e..90b4b39582b 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py @@ -1,3 +1,4 @@ +import json import httpx from unittest.mock import patch, PropertyMock @@ -267,7 +268,7 @@ def test_validate_chunk_without_usage_keeps_none(): assert chunk.usage is None -def test_validated_usage_survives_nested_model_dump(): +def test_validated_usage_survives_nested_model_dump_json(): from litellm.llms.sap.chat.handler import _StreamParser from litellm.types.utils import ModelResponseStream @@ -276,7 +277,7 @@ def test_validated_usage_survives_nested_model_dump(): model_response = ModelResponseStream() setattr(model_response, "usage", chunk.usage) - dumped = model_response.model_dump() + dumped = json.loads(model_response.model_dump_json()) assert dumped["usage"]["total_tokens"] == 62528 From e4febc228db4a59dfd9f0cfa5c9da60a428d3ab4 Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Mon, 14 Sep 2026 11:26:31 +0800 Subject: [PATCH 10/11] test(sap): cover the non-dict final_result branch The isinstance guard added in the previous commit left one line uncovered, so patch coverage reported a miss on the new branch. A final_result that is not a dict is treated as an unknown event and skipped instead of reaching model_validate, and this pins that behavior. --- tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py b/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py index 90b4b39582b..ee9589966a1 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py @@ -315,3 +315,10 @@ def test_to_openai_chunk_from_orchestration_result(): assert chunk is not None assert chunk.choices[0].delta.content == "Hello " assert chunk.choices[0].logprobs is None + + +def test_to_openai_chunk_ignores_non_dict_final_result(): + from litellm.llms.sap.chat.handler import _StreamParser + + assert _StreamParser.to_openai_chunk({"final_result": None}) is None + assert _StreamParser.to_openai_chunk({"final_result": "not-a-chunk"}) is None From 035f68a8f164789e89749eab2651a169961ec5af Mon Sep 17 00:00:00 2001 From: "ZOU Yi (BD/SWD-WDE1)" Date: Wed, 16 Sep 2026 17:49:32 +0800 Subject: [PATCH 11/11] chore(ci): retrigger CI Both remaining failures come from the base branch, not this PR: the Vertex cost asserts depend on which price map litellm loads, and proxy-behavior is a known flake. Re-running to get a fresh set of results.