Merge 035f68a8f1 into 252c71c0b2

litellm/llms/sap/chat/handler.py

@@ -9,6 +9,7 @@ import httpx
 
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.types.llms.openai import OpenAIChatCompletionChunk
+from litellm.types.utils import Usage
 
 from ...custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 
@@ -46,6 +47,20 @@ def _is_terminal_chunk(chunk: OpenAIChatCompletionChunk) -> bool:
 class _StreamParser:
     """Normalize orchestration streaming events into OpenAI-like chunks."""
 
+    @staticmethod
+    def _validate_chunk(
+        payload: dict[str, object],  # mutable-ok: normalized in place (pops empty logprobs) before validation
+    ) -> OpenAIChatCompletionChunk:
+        choices: Final = payload.get("choices")
+        if isinstance(choices, list):
+            for choice in choices:
+                if isinstance(choice, dict) and not choice.get("logprobs"):
+                    choice.pop("logprobs", None)  # mutable-ok: pops the logprobs key in-place before model_validate
+        chunk = OpenAIChatCompletionChunk.model_validate(payload)
+        if chunk.usage is not None:
+            chunk.usage = Usage.model_validate(chunk.usage.model_dump())
+        return chunk
+
     @staticmethod
     def _from_orchestration_result(evt: dict) -> OpenAIChatCompletionChunk | None:
         """
@@ -55,25 +70,24 @@ class _StreamParser:
         if not orc:
             return None
 
-        return OpenAIChatCompletionChunk.model_validate(
-            {
-                "id": orc.get("id") or evt.get("request_id") or "stream-chunk",
-                "object": orc.get("object") or "chat.completion.chunk",
-                "created": orc.get("created") or evt.get("created") or _now_ts(),
-                "model": orc.get("model") or "unknown",
-                "choices": [
-                    {
-                        "index": c.get("index", 0),
-                        "delta": c.get("delta") or {},
-                        "finish_reason": c.get("finish_reason"),
-                    }
-                    for c in (orc.get("choices") or [])
-                ],
-            }
-        )
+        payload: Final[dict[str, object]] = {
+            "id": orc.get("id") or evt.get("request_id") or "stream-chunk",
+            "object": orc.get("object") or "chat.completion.chunk",
+            "created": orc.get("created") or evt.get("created") or _now_ts(),
+            "model": orc.get("model") or "unknown",
+            "choices": [
+                {
+                    "index": c.get("index", 0),
+                    "delta": c.get("delta") or {},
+                    "finish_reason": c.get("finish_reason"),
+                }
+                for c in (orc.get("choices") or [])
+            ],
+        }
+        return _StreamParser._validate_chunk(payload)
 
     @staticmethod
-    def to_openai_chunk(event_obj: dict) -> OpenAIChatCompletionChunk | None:
+    def to_openai_chunk(event_obj: dict[str, object]) -> OpenAIChatCompletionChunk | None:
         """
         Accepts:
           - {"final_result": <openai-style CHUNK>}   (IMPORTANT: this is just another chunk, NOT terminal)
@@ -89,11 +103,14 @@ class _StreamParser:
 
         # FINAL RESULT IS *NOT* TERMINAL: treat it as the next chunk
         if "final_result" in event_obj:
-            fr: Final = event_obj["final_result"] or {}
+            final_result: Final = event_obj["final_result"]
+            if not isinstance(final_result, dict):
+                return None
+            fr: Final[dict[str, object]] = final_result
             # ensure it looks like an OpenAI chunk
             if "object" not in fr:
                 fr["object"] = "chat.completion.chunk"
-            return OpenAIChatCompletionChunk.model_validate(fr)
+            return _StreamParser._validate_chunk(fr)
 
         # Orchestration incremental delta
         if "orchestration_result" in event_obj:
@@ -101,7 +118,7 @@ class _StreamParser:
 
         # Already an OpenAI-like chunk
         if "choices" in event_obj and "object" in event_obj:
-            return OpenAIChatCompletionChunk.model_validate(event_obj)
+            return _StreamParser._validate_chunk(event_obj)
 
         # Unknown / heartbeat / metrics
         return None

litellm/llms/sap/chat/transformation.py

@@ -18,6 +18,7 @@ if TYPE_CHECKING:
     import tiktoken
 
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
     LiteLLMLoggingObj = _LiteLLMLoggingObj
 else:
@@ -136,6 +137,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
         self.token_creator = None
         self._base_url = None
         self._resource_group = None
+        self._http_client: HTTPHandler | None = None
 
     def run_env_setup(self, service_key: str | None = None) -> None:
         try:
@@ -169,9 +171,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
 
     @cached_property
     def deployment_url(self) -> str:
-        # Keep a short, tight client lifecycle here to avoid fd leaks
-        client: Final = litellm.module_level_client
-        # with httpx.Client(timeout=30) as client:
+        client: Final = self._http_client if self._http_client is not None else litellm.module_level_client
         deployments: Final = client.get(f"{self.base_url}/lm/deployments", headers=self.headers).json()
         valid: Final[list[tuple[str, str]]] = []
         for dep in deployments.get("resources", []):
@@ -183,6 +183,15 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
                 if cfg.get("executableId") == "orchestration":
                     valid.append((dep["deploymentUrl"], dep["createdAt"]))
             # newest first
+        if not valid:
+            raise GenAIHubOrchestrationError(
+                status_code=404,
+                message=(
+                    "No orchestration deployment found in SAP AI Core resource group "
+                    f"'{self.resource_group}'. Create/start an orchestration deployment "
+                    "in SAP AI Launchpad, then retry."
+                ),
+            )
         return sorted(valid, key=lambda x: x[1], reverse=True)[0][0]
 
     @classmethod

tests/test_litellm/llms/sap/chat/test_sap_chat_calls.py

@@ -1,3 +1,4 @@
+import json
 import httpx
 from unittest.mock import patch, PropertyMock
 
@@ -204,3 +205,120 @@ async def test_sap_chat_required_headers(
                 f"Header '{header_name}' has incorrect value. "
                 f"Expected: '{expected_value}', Got: '{request.headers[header_name]}'"
             )
+
+
+def _final_chunk_payload() -> dict:
+    return {
+        "id": "chatcmpl-sap-final",
+        "object": "chat.completion.chunk",
+        "created": 1761319270,
+        "model": "anthropic--claude-4.7-opus",
+        "choices": [
+            {
+                "index": 0,
+                "delta": {},
+                "logprobs": {},
+                "finish_reason": "tool_calls",
+            }
+        ],
+        "usage": {
+            "completion_tokens": 206,
+            "prompt_tokens": 62322,
+            "total_tokens": 62528,
+        },
+    }
+
+
+def test_validate_chunk_drops_empty_logprobs():
+    from litellm.llms.sap.chat.handler import _StreamParser
+
+    chunk = _StreamParser._validate_chunk(_final_chunk_payload())
+    assert chunk.choices[0].logprobs is None
+
+
+def test_validate_chunk_preserves_real_logprobs():
+    from litellm.llms.sap.chat.handler import _StreamParser
+
+    payload = _final_chunk_payload()
+    payload["choices"][0]["logprobs"] = {
+        "content": [{"token": "Hello", "logprob": -0.1, "bytes": None, "top_logprobs": []}]
+    }
+    chunk = _StreamParser._validate_chunk(payload)
+    assert chunk.choices[0].logprobs is not None
+    assert chunk.choices[0].logprobs.content[0].token == "Hello"
+
+
+def test_validate_chunk_converts_usage_to_litellm_usage():
+    from litellm.llms.sap.chat.handler import _StreamParser
+    from litellm.types.utils import Usage
+
+    chunk = _StreamParser._validate_chunk(_final_chunk_payload())
+    assert isinstance(chunk.usage, Usage)
+    assert chunk.usage.completion_tokens == 206
+    assert chunk.usage.prompt_tokens == 62322
+    assert chunk.usage.total_tokens == 62528
+
+
+def test_validate_chunk_without_usage_keeps_none():
+    from litellm.llms.sap.chat.handler import _StreamParser
+
+    payload = _final_chunk_payload()
+    del payload["usage"]
+    chunk = _StreamParser._validate_chunk(payload)
+    assert chunk.usage is None
+
+
+def test_validated_usage_survives_nested_model_dump_json():
+    from litellm.llms.sap.chat.handler import _StreamParser
+    from litellm.types.utils import ModelResponseStream
+
+    chunk = _StreamParser._validate_chunk(_final_chunk_payload())
+
+    model_response = ModelResponseStream()
+    setattr(model_response, "usage", chunk.usage)
+
+    dumped = json.loads(model_response.model_dump_json())
+    assert dumped["usage"]["total_tokens"] == 62528
+
+
+def test_to_openai_chunk_normalizes_openai_shaped_event():
+    from litellm.llms.sap.chat.handler import _StreamParser
+    from litellm.types.utils import Usage
+
+    chunk = _StreamParser.to_openai_chunk(_final_chunk_payload())
+    assert chunk is not None
+    assert chunk.choices[0].logprobs is None
+    assert isinstance(chunk.usage, Usage)
+
+
+def test_to_openai_chunk_from_orchestration_result():
+    from litellm.llms.sap.chat.handler import _StreamParser
+
+    event = {
+        "request_id": "a07127d3-cb74-9427-a4dc-ef9bf424fb43",
+        "orchestration_result": {
+            "id": "chatcmpl-sap-delta",
+            "object": "chat.completion.chunk",
+            "created": 1761319270,
+            "model": "anthropic--claude-4.7-opus",
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": {"role": "assistant", "content": "Hello "},
+                    "logprobs": {},
+                    "finish_reason": None,
+                }
+            ],
+        },
+    }
+    chunk = _StreamParser.to_openai_chunk(event)
+    assert chunk is not None
+    assert chunk.choices[0].delta.content == "Hello "
+    assert chunk.choices[0].logprobs is None
+
+
+def test_to_openai_chunk_ignores_non_dict_final_result():
+    from litellm.llms.sap.chat.handler import _StreamParser
+
+    assert _StreamParser.to_openai_chunk({"final_result": None}) is None
+    assert _StreamParser.to_openai_chunk({"final_result": "not-a-chunk"}) is None

tests/test_litellm/llms/sap/chat/test_sap_transformation.py

@@ -639,3 +639,18 @@ class TestSAPTransformationIntegration:
                 config["config"]["modules"][1]["translation"]["input"]["type"]
                 == "sap_document_translation"
             )
+
+    def test_deployment_url_raises_404_when_no_orchestration_deployment(self, mock_config):
+        from unittest.mock import MagicMock
+
+        from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError
+
+        mock_client = MagicMock()
+        mock_client.get.return_value.json.return_value = {"resources": []}
+        mock_config._http_client = mock_client
+
+        with pytest.raises(GenAIHubOrchestrationError) as exc_info:
+            _ = mock_config.deployment_url
+
+        assert exc_info.value.status_code == 404
+        assert "test-group" in exc_info.value.message