diff --git a/litellm/llms/sap/chat/handler.py b/litellm/llms/sap/chat/handler.py index b4f4d4faddb..d0be0086214 100755 --- a/litellm/llms/sap/chat/handler.py +++ b/litellm/llms/sap/chat/handler.py @@ -9,6 +9,7 @@ import httpx from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.openai import OpenAIChatCompletionChunk +from litellm.types.utils import Usage from ...custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -46,6 +47,20 @@ def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool: class _StreamParser: """Normalize orchestration streaming events into OpenAI-like chunks.""" + @staticmethod + def _validate_chunk( + payload: dict[str, object], # mutable-ok: normalized in place (pops empty logprobs) before validation + ) -> OpenAIChatCompletionChunk: + choices: Final = payload.get("choices") + if isinstance(choices, list): + for choice in choices: + if isinstance(choice, dict) and not choice.get("logprobs"): + choice.pop("logprobs", None) # mutable-ok: pops the logprobs key in-place before model_validate + chunk = OpenAIChatCompletionChunk.model_validate(payload) + if chunk.usage is not None: + chunk.usage = Usage.model_validate(chunk.usage.model_dump()) + return chunk + @staticmethod def _from_orchestration_result(evt: dict) -> OpenAIChatCompletionChunk | None: """ @@ -55,25 +70,24 @@ class _StreamParser: if not orc: return None - return OpenAIChatCompletionChunk.model_validate( - { - "id": orc.get("id") or evt.get("request_id") or "stream-chunk", - "object": orc.get("object") or "chat.completion.chunk", - "created": orc.get("created") or evt.get("created") or _now_ts(), - "model": orc.get("model") or "unknown", - "choices": [ - { - "index": c.get("index", 0), - "delta": c.get("delta") or {}, - "finish_reason": c.get("finish_reason"), - } - for c in (orc.get("choices") or []) - ], - } - ) + payload: Final[dict[str, object]] = { + "id": orc.get("id") or evt.get("request_id") or "stream-chunk", + "object": orc.get("object") or "chat.completion.chunk", + "created": orc.get("created") or evt.get("created") or _now_ts(), + "model": orc.get("model") or "unknown", + "choices": [ + { + "index": c.get("index", 0), + "delta": c.get("delta") or {}, + "finish_reason": c.get("finish_reason"), + } + for c in (orc.get("choices") or []) + ], + } + return _StreamParser._validate_chunk(payload) @staticmethod - def to_openai_chunk(event_obj: dict) -> OpenAIChatCompletionChunk | None: + def to_openai_chunk(event_obj: dict[str, object]) -> OpenAIChatCompletionChunk | None: """ Accepts: - {"final_result": } (IMPORTANT: this is just another chunk, NOT terminal) @@ -89,11 +103,14 @@ class _StreamParser: # FINAL RESULT IS *NOT* TERMINAL: treat it as the next chunk if "final_result" in event_obj: - fr: Final = event_obj["final_result"] or {} + final_result: Final = event_obj["final_result"] + if not isinstance(final_result, dict): + return None + fr: Final[dict[str, object]] = final_result # ensure it looks like an OpenAI chunk if "object" not in fr: fr["object"] = "chat.completion.chunk" - return OpenAIChatCompletionChunk.model_validate(fr) + return _StreamParser._validate_chunk(fr) # Orchestration incremental delta if "orchestration_result" in event_obj: @@ -101,7 +118,7 @@ class _StreamParser: # Already an OpenAI-like chunk if "choices" in event_obj and "object" in event_obj: - return OpenAIChatCompletionChunk.model_validate(event_obj) + return _StreamParser._validate_chunk(event_obj) # Unknown / heartbeat / metrics return None diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index d64d7a57281..f41a4c3f336 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: import tiktoken from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import HTTPHandler LiteLLMLoggingObj = _LiteLLMLoggingObj else: @@ -136,6 +137,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): self.token_creator = None self._base_url = None self._resource_group = None + self._http_client: HTTPHandler | None = None def run_env_setup(self, service_key: str | None = None) -> None: try: @@ -169,9 +171,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): @cached_property def deployment_url(self) -> str: - # Keep a short, tight client lifecycle here to avoid fd leaks - client: Final = litellm.module_level_client - # with httpx.Client(timeout=30) as client: + client: Final = self._http_client if self._http_client is not None else litellm.module_level_client deployments: Final = client.get(f"{self.base_url}/lm/deployments", headers=self.headers).json() valid: Final[list[tuple[str, str]]] = [] for dep in deployments.get("resources", []): @@ -183,6 +183,15 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): if cfg.get("executableId") == "orchestration": valid.append((dep["deploymentUrl"], dep["createdAt"])) # newest first + if not valid: + raise GenAIHubOrchestrationError( + status_code=404, + message=( + "No orchestration deployment found in SAP AI Core resource group " + f"'{self.resource_group}'. Create/start an orchestration deployment " + "in SAP AI Launchpad, then retry." + ), + ) return sorted(valid, key=lambda x: x[1], reverse=True)[0][0] @classmethod diff --git a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py b/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py index c41254f1a2b..ee9589966a1 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py @@ -1,3 +1,4 @@ +import json import httpx from unittest.mock import patch, PropertyMock @@ -204,3 +205,120 @@ async def test_sap_chat_required_headers( f"Header '{header_name}' has incorrect value. " f"Expected: '{expected_value}', Got: '{request.headers[header_name]}'" ) + + +def _final_chunk_payload() -> dict: + return { + "id": "chatcmpl-sap-final", + "object": "chat.completion.chunk", + "created": 1761319270, + "model": "anthropic--claude-4.7-opus", + "choices": [ + { + "index": 0, + "delta": {}, + "logprobs": {}, + "finish_reason": "tool_calls", + } + ], + "usage": { + "completion_tokens": 206, + "prompt_tokens": 62322, + "total_tokens": 62528, + }, + } + + +def test_validate_chunk_drops_empty_logprobs(): + from litellm.llms.sap.chat.handler import _StreamParser + + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + assert chunk.choices[0].logprobs is None + + +def test_validate_chunk_preserves_real_logprobs(): + from litellm.llms.sap.chat.handler import _StreamParser + + payload = _final_chunk_payload() + payload["choices"][0]["logprobs"] = { + "content": [{"token": "Hello", "logprob": -0.1, "bytes": None, "top_logprobs": []}] + } + chunk = _StreamParser._validate_chunk(payload) + assert chunk.choices[0].logprobs is not None + assert chunk.choices[0].logprobs.content[0].token == "Hello" + + +def test_validate_chunk_converts_usage_to_litellm_usage(): + from litellm.llms.sap.chat.handler import _StreamParser + from litellm.types.utils import Usage + + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + assert isinstance(chunk.usage, Usage) + assert chunk.usage.completion_tokens == 206 + assert chunk.usage.prompt_tokens == 62322 + assert chunk.usage.total_tokens == 62528 + + +def test_validate_chunk_without_usage_keeps_none(): + from litellm.llms.sap.chat.handler import _StreamParser + + payload = _final_chunk_payload() + del payload["usage"] + chunk = _StreamParser._validate_chunk(payload) + assert chunk.usage is None + + +def test_validated_usage_survives_nested_model_dump_json(): + from litellm.llms.sap.chat.handler import _StreamParser + from litellm.types.utils import ModelResponseStream + + chunk = _StreamParser._validate_chunk(_final_chunk_payload()) + + model_response = ModelResponseStream() + setattr(model_response, "usage", chunk.usage) + + dumped = json.loads(model_response.model_dump_json()) + assert dumped["usage"]["total_tokens"] == 62528 + + +def test_to_openai_chunk_normalizes_openai_shaped_event(): + from litellm.llms.sap.chat.handler import _StreamParser + from litellm.types.utils import Usage + + chunk = _StreamParser.to_openai_chunk(_final_chunk_payload()) + assert chunk is not None + assert chunk.choices[0].logprobs is None + assert isinstance(chunk.usage, Usage) + + +def test_to_openai_chunk_from_orchestration_result(): + from litellm.llms.sap.chat.handler import _StreamParser + + event = { + "request_id": "a07127d3-cb74-9427-a4dc-ef9bf424fb43", + "orchestration_result": { + "id": "chatcmpl-sap-delta", + "object": "chat.completion.chunk", + "created": 1761319270, + "model": "anthropic--claude-4.7-opus", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": "Hello "}, + "logprobs": {}, + "finish_reason": None, + } + ], + }, + } + chunk = _StreamParser.to_openai_chunk(event) + assert chunk is not None + assert chunk.choices[0].delta.content == "Hello " + assert chunk.choices[0].logprobs is None + + +def test_to_openai_chunk_ignores_non_dict_final_result(): + from litellm.llms.sap.chat.handler import _StreamParser + + assert _StreamParser.to_openai_chunk({"final_result": None}) is None + assert _StreamParser.to_openai_chunk({"final_result": "not-a-chunk"}) is None diff --git a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py index 3601bdd0d5e..565545a8d84 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py @@ -639,3 +639,18 @@ class TestSAPTransformationIntegration: config["config"]["modules"][1]["translation"]["input"]["type"] == "sap_document_translation" ) + + def test_deployment_url_raises_404_when_no_orchestration_deployment(self, mock_config): + from unittest.mock import MagicMock + + from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError + + mock_client = MagicMock() + mock_client.get.return_value.json.return_value = {"resources": []} + mock_config._http_client = mock_client + + with pytest.raises(GenAIHubOrchestrationError) as exc_info: + _ = mock_config.deployment_url + + assert exc_info.value.status_code == 404 + assert "test-group" in exc_info.value.message