From 79b1dd7e0bb90b7366f3d931fc500d1033c222bf Mon Sep 17 00:00:00 2001 From: Yamac Ay Date: Thu, 25 Jun 2026 10:56:01 +0200 Subject: [PATCH] feat(sap): support multiple orchestration deployments via name filter When a BTP tenant has more than one orchestration deployment, the previous code silently picked the newest by creation date with no warning and no way to override it. This led to the wrong deployment (e.g. one with Grounding pre-configured) being used. Three changes: - Warn via verbose_logger when >1 valid deployment is found, naming the chosen deployment and listing the others. - Honour SAP_ORCHESTRATION_DEPLOYMENT_NAME env var to filter by configuration name globally. - Accept deployment_name in litellm_params for per-call selection; takes priority over the env var. Also raises GenAIHubOrchestrationError with a clear message when no deployment is found, or when the requested name matches nothing (listing available names in the error). --- litellm/llms/sap/chat/transformation.py | 100 +++++++++++----- .../llms/sap/chat/test_sap_transformation.py | 113 ++++++++++++++++++ 2 files changed, 185 insertions(+), 28 deletions(-) diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 4bf8272a334..51cb734be34 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -2,23 +2,25 @@ Translate from OpenAI's `/v1/chat/completions` to SAP Generative AI Hub's Orchestration Service`v2/completion` """ +import os +from functools import cached_property from typing import ( + Any, + AsyncIterator, + Dict, + FrozenSet, + Iterator, List, Optional, - Union, - Dict, Tuple, - Any, TYPE_CHECKING, - Iterator, - AsyncIterator, - FrozenSet, + Union, ) -from functools import cached_property -import litellm + import httpx +import litellm - +from litellm._logging import verbose_logger from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -32,6 +34,11 @@ else: LiteLLMLoggingObj = Any from ..credentials import get_token_creator +from .handler import ( + AsyncSAPStreamIterator, + GenAIHubOrchestrationError, + SAPStreamIterator, +) from .models import ( ChatCompletionTool, OrchestrationRequest, @@ -42,11 +49,6 @@ from .models import ( SAPToolChatMessage, SAPUserMessage, ) -from .handler import ( - GenAIHubOrchestrationError, - AsyncSAPStreamIterator, - SAPStreamIterator, -) # Keys routed outside SAP orchestration `model.params` (prompt, stream, fallbacks, etc.) _SAP_MODEL_PARAMS_EXCLUDED_KEYS: FrozenSet[str] = frozenset( @@ -177,20 +179,58 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): @cached_property def deployment_url(self) -> str: - # Keep a short, tight client lifecycle here to avoid fd leaks + return self._resolve_deployment_url(deployment_name=None) + + def _resolve_deployment_url(self, deployment_name: Optional[str]) -> str: client = litellm.module_level_client - # with httpx.Client(timeout=30) as client: - deployments = client.get(f"{self.base_url}/lm/deployments", headers=self.headers).json() - valid: List[Tuple[str, str]] = [] + deployments = client.get( + f"{self.base_url}/lm/deployments", headers=self.headers + ).json() + + valid: List[Tuple[str, str, str]] = [] for dep in deployments.get("resources", []): - if dep.get("scenarioId") == "orchestration": - cfg = client.get( - f"{self.base_url}/lm/configurations/{dep['configurationId']}", - headers=self.headers, - ).json() - if cfg.get("executableId") == "orchestration": - valid.append((dep["deploymentUrl"], dep["createdAt"])) - # newest first + if dep.get("scenarioId") != "orchestration": + continue + cfg = client.get( + f'{self.base_url}/lm/configurations/{dep["configurationId"]}', + headers=self.headers, + ).json() + if cfg.get("executableId") == "orchestration": + valid.append((dep["deploymentUrl"], dep["createdAt"], cfg.get("name", ""))) + + if not valid: + raise GenAIHubOrchestrationError( + status_code=404, + message="No orchestration deployment found in SAP AI Core.", + ) + + name_filter = deployment_name or os.environ.get("SAP_ORCHESTRATION_DEPLOYMENT_NAME") + if name_filter: + filtered = [v for v in valid if v[2] == name_filter] + if not filtered: + available = [v[2] for v in valid] + raise GenAIHubOrchestrationError( + status_code=404, + message=( + f"No orchestration deployment named {name_filter!r} found. " + f"Available: {available}" + ), + ) + valid = filtered + + if len(valid) > 1: + sorted_valid = sorted(valid, key=lambda x: x[1], reverse=True) + chosen = sorted_valid[0] + others = [v[2] or v[0] for v in sorted_valid[1:]] + verbose_logger.warning( + f"SAP: found {len(valid)} orchestration deployments; using the newest one " + f"(name={chosen[2]!r}, url={chosen[0]!r}). " + f"Others ignored: {others}. " + "Set SAP_ORCHESTRATION_DEPLOYMENT_NAME to select a specific one, " + "or use the deployment_name per-call parameter." + ) + return chosen[0] + return sorted(valid, key=lambda x: x[1], reverse=True)[0][0] @classmethod @@ -258,8 +298,12 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): litellm_params: dict, stream: Optional[bool] = None, ): - api_base_ = f"{self.deployment_url}/v2/completion" - return api_base_ + deployment_name = litellm_params.get("deployment_name") + if deployment_name: + base = self._resolve_deployment_url(deployment_name=deployment_name) + else: + base = self.deployment_url + return f"{base}/v2/completion" def _build_prompt_module( self, diff --git a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py index 3601bdd0d5e..d488a4071b1 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py @@ -1,5 +1,6 @@ import warnings import pytest +from unittest.mock import patch, MagicMock from pydantic import ValidationError @@ -639,3 +640,115 @@ class TestSAPTransformationIntegration: config["config"]["modules"][1]["translation"]["input"]["type"] == "sap_document_translation" ) + + +def _make_config(): + from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig + + config = GenAIHubOrchestrationConfig() + config.token_creator = lambda: "Bearer TEST_TOKEN" + config._base_url = "https://api.test-sap.com" + config._resource_group = "test-group" + return config + + +def _mock_deployments(*names: str): + """Return a mock httpx client whose GET responses simulate AI Core deployment/config APIs.""" + deployments_payload = { + "resources": [ + {"scenarioId": "orchestration", "configurationId": f"cfg-{n}", "deploymentUrl": f"https://deploy-{n}.sap.com", "createdAt": f"2024-01-0{i+1}T00:00:00Z"} + for i, n in enumerate(names) + ] + } + configs = { + f"cfg-{n}": {"executableId": "orchestration", "name": n} + for n in names + } + + def fake_get(url, headers=None): + resp = MagicMock() + if "/lm/deployments" in url: + resp.json.return_value = deployments_payload + else: + cfg_id = url.split("/")[-1] + resp.json.return_value = configs.get(cfg_id, {}) + return resp + + client = MagicMock() + client.get.side_effect = fake_get + return client + + +class TestDeploymentSelection: + def test_single_deployment_returns_its_url(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("my-orch")): + url = config._resolve_deployment_url(deployment_name=None) + assert url == "https://deploy-my-orch.sap.com" + + def test_multiple_deployments_picks_newest(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("older", "newer")): + url = config._resolve_deployment_url(deployment_name=None) + assert url == "https://deploy-newer.sap.com" + + def test_multiple_deployments_emits_warning(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("older", "newer")): + with patch("litellm.llms.sap.chat.transformation.verbose_logger") as mock_log: + config._resolve_deployment_url(deployment_name=None) + mock_log.warning.assert_called_once() + msg = mock_log.warning.call_args[0][0] + assert "2 orchestration deployments" in msg + assert "SAP_ORCHESTRATION_DEPLOYMENT_NAME" in msg + + def test_deployment_name_arg_selects_by_name(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("grounding-orch", "plain-orch")): + url = config._resolve_deployment_url(deployment_name="grounding-orch") + assert url == "https://deploy-grounding-orch.sap.com" + + def test_env_var_selects_by_name(self, monkeypatch): + monkeypatch.setenv("SAP_ORCHESTRATION_DEPLOYMENT_NAME", "plain-orch") + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("grounding-orch", "plain-orch")): + url = config._resolve_deployment_url(deployment_name=None) + assert url == "https://deploy-plain-orch.sap.com" + + def test_deployment_name_arg_takes_priority_over_env_var(self, monkeypatch): + monkeypatch.setenv("SAP_ORCHESTRATION_DEPLOYMENT_NAME", "plain-orch") + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("grounding-orch", "plain-orch")): + url = config._resolve_deployment_url(deployment_name="grounding-orch") + assert url == "https://deploy-grounding-orch.sap.com" + + def test_unknown_name_raises_with_available_list(self): + from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError + + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("alpha", "beta")): + with pytest.raises(GenAIHubOrchestrationError) as exc_info: + config._resolve_deployment_url(deployment_name="gamma") + assert "gamma" in str(exc_info.value) + assert "alpha" in str(exc_info.value) or "beta" in str(exc_info.value) + + def test_no_deployments_raises(self): + from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError + + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments()): + with pytest.raises(GenAIHubOrchestrationError) as exc_info: + config._resolve_deployment_url(deployment_name=None) + assert "No orchestration deployment found" in str(exc_info.value) + + def test_get_complete_url_uses_deployment_name_from_litellm_params(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("grounding-orch", "plain-orch")): + url = config.get_complete_url( + api_base=None, + api_key=None, + model="gpt-4o", + optional_params={}, + litellm_params={"deployment_name": "plain-orch"}, + ) + assert url == "https://deploy-plain-orch.sap.com/v2/completion"