diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 4bf8272a334..51cb734be34 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -2,23 +2,25 @@ Translate from OpenAI's `/v1/chat/completions` to SAP Generative AI Hub's Orchestration Service`v2/completion` """ +import os +from functools import cached_property from typing import ( + Any, + AsyncIterator, + Dict, + FrozenSet, + Iterator, List, Optional, - Union, - Dict, Tuple, - Any, TYPE_CHECKING, - Iterator, - AsyncIterator, - FrozenSet, + Union, ) -from functools import cached_property -import litellm + import httpx +import litellm - +from litellm._logging import verbose_logger from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -32,6 +34,11 @@ else: LiteLLMLoggingObj = Any from ..credentials import get_token_creator +from .handler import ( + AsyncSAPStreamIterator, + GenAIHubOrchestrationError, + SAPStreamIterator, +) from .models import ( ChatCompletionTool, OrchestrationRequest, @@ -42,11 +49,6 @@ from .models import ( SAPToolChatMessage, SAPUserMessage, ) -from .handler import ( - GenAIHubOrchestrationError, - AsyncSAPStreamIterator, - SAPStreamIterator, -) # Keys routed outside SAP orchestration `model.params` (prompt, stream, fallbacks, etc.) _SAP_MODEL_PARAMS_EXCLUDED_KEYS: FrozenSet[str] = frozenset( @@ -177,20 +179,58 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): @cached_property def deployment_url(self) -> str: - # Keep a short, tight client lifecycle here to avoid fd leaks + return self._resolve_deployment_url(deployment_name=None) + + def _resolve_deployment_url(self, deployment_name: Optional[str]) -> str: client = litellm.module_level_client - # with httpx.Client(timeout=30) as client: - deployments = client.get(f"{self.base_url}/lm/deployments", headers=self.headers).json() - valid: List[Tuple[str, str]] = [] + deployments = client.get( + f"{self.base_url}/lm/deployments", headers=self.headers + ).json() + + valid: List[Tuple[str, str, str]] = [] for dep in deployments.get("resources", []): - if dep.get("scenarioId") == "orchestration": - cfg = client.get( - f"{self.base_url}/lm/configurations/{dep['configurationId']}", - headers=self.headers, - ).json() - if cfg.get("executableId") == "orchestration": - valid.append((dep["deploymentUrl"], dep["createdAt"])) - # newest first + if dep.get("scenarioId") != "orchestration": + continue + cfg = client.get( + f'{self.base_url}/lm/configurations/{dep["configurationId"]}', + headers=self.headers, + ).json() + if cfg.get("executableId") == "orchestration": + valid.append((dep["deploymentUrl"], dep["createdAt"], cfg.get("name", ""))) + + if not valid: + raise GenAIHubOrchestrationError( + status_code=404, + message="No orchestration deployment found in SAP AI Core.", + ) + + name_filter = deployment_name or os.environ.get("SAP_ORCHESTRATION_DEPLOYMENT_NAME") + if name_filter: + filtered = [v for v in valid if v[2] == name_filter] + if not filtered: + available = [v[2] for v in valid] + raise GenAIHubOrchestrationError( + status_code=404, + message=( + f"No orchestration deployment named {name_filter!r} found. " + f"Available: {available}" + ), + ) + valid = filtered + + if len(valid) > 1: + sorted_valid = sorted(valid, key=lambda x: x[1], reverse=True) + chosen = sorted_valid[0] + others = [v[2] or v[0] for v in sorted_valid[1:]] + verbose_logger.warning( + f"SAP: found {len(valid)} orchestration deployments; using the newest one " + f"(name={chosen[2]!r}, url={chosen[0]!r}). " + f"Others ignored: {others}. " + "Set SAP_ORCHESTRATION_DEPLOYMENT_NAME to select a specific one, " + "or use the deployment_name per-call parameter." + ) + return chosen[0] + return sorted(valid, key=lambda x: x[1], reverse=True)[0][0] @classmethod @@ -258,8 +298,12 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): litellm_params: dict, stream: Optional[bool] = None, ): - api_base_ = f"{self.deployment_url}/v2/completion" - return api_base_ + deployment_name = litellm_params.get("deployment_name") + if deployment_name: + base = self._resolve_deployment_url(deployment_name=deployment_name) + else: + base = self.deployment_url + return f"{base}/v2/completion" def _build_prompt_module( self, diff --git a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py index 3601bdd0d5e..d488a4071b1 100644 --- a/tests/test_litellm/llms/sap/chat/test_sap_transformation.py +++ b/tests/test_litellm/llms/sap/chat/test_sap_transformation.py @@ -1,5 +1,6 @@ import warnings import pytest +from unittest.mock import patch, MagicMock from pydantic import ValidationError @@ -639,3 +640,115 @@ class TestSAPTransformationIntegration: config["config"]["modules"][1]["translation"]["input"]["type"] == "sap_document_translation" ) + + +def _make_config(): + from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig + + config = GenAIHubOrchestrationConfig() + config.token_creator = lambda: "Bearer TEST_TOKEN" + config._base_url = "https://api.test-sap.com" + config._resource_group = "test-group" + return config + + +def _mock_deployments(*names: str): + """Return a mock httpx client whose GET responses simulate AI Core deployment/config APIs.""" + deployments_payload = { + "resources": [ + {"scenarioId": "orchestration", "configurationId": f"cfg-{n}", "deploymentUrl": f"https://deploy-{n}.sap.com", "createdAt": f"2024-01-0{i+1}T00:00:00Z"} + for i, n in enumerate(names) + ] + } + configs = { + f"cfg-{n}": {"executableId": "orchestration", "name": n} + for n in names + } + + def fake_get(url, headers=None): + resp = MagicMock() + if "/lm/deployments" in url: + resp.json.return_value = deployments_payload + else: + cfg_id = url.split("/")[-1] + resp.json.return_value = configs.get(cfg_id, {}) + return resp + + client = MagicMock() + client.get.side_effect = fake_get + return client + + +class TestDeploymentSelection: + def test_single_deployment_returns_its_url(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("my-orch")): + url = config._resolve_deployment_url(deployment_name=None) + assert url == "https://deploy-my-orch.sap.com" + + def test_multiple_deployments_picks_newest(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("older", "newer")): + url = config._resolve_deployment_url(deployment_name=None) + assert url == "https://deploy-newer.sap.com" + + def test_multiple_deployments_emits_warning(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("older", "newer")): + with patch("litellm.llms.sap.chat.transformation.verbose_logger") as mock_log: + config._resolve_deployment_url(deployment_name=None) + mock_log.warning.assert_called_once() + msg = mock_log.warning.call_args[0][0] + assert "2 orchestration deployments" in msg + assert "SAP_ORCHESTRATION_DEPLOYMENT_NAME" in msg + + def test_deployment_name_arg_selects_by_name(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("grounding-orch", "plain-orch")): + url = config._resolve_deployment_url(deployment_name="grounding-orch") + assert url == "https://deploy-grounding-orch.sap.com" + + def test_env_var_selects_by_name(self, monkeypatch): + monkeypatch.setenv("SAP_ORCHESTRATION_DEPLOYMENT_NAME", "plain-orch") + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("grounding-orch", "plain-orch")): + url = config._resolve_deployment_url(deployment_name=None) + assert url == "https://deploy-plain-orch.sap.com" + + def test_deployment_name_arg_takes_priority_over_env_var(self, monkeypatch): + monkeypatch.setenv("SAP_ORCHESTRATION_DEPLOYMENT_NAME", "plain-orch") + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("grounding-orch", "plain-orch")): + url = config._resolve_deployment_url(deployment_name="grounding-orch") + assert url == "https://deploy-grounding-orch.sap.com" + + def test_unknown_name_raises_with_available_list(self): + from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError + + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("alpha", "beta")): + with pytest.raises(GenAIHubOrchestrationError) as exc_info: + config._resolve_deployment_url(deployment_name="gamma") + assert "gamma" in str(exc_info.value) + assert "alpha" in str(exc_info.value) or "beta" in str(exc_info.value) + + def test_no_deployments_raises(self): + from litellm.llms.sap.chat.handler import GenAIHubOrchestrationError + + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments()): + with pytest.raises(GenAIHubOrchestrationError) as exc_info: + config._resolve_deployment_url(deployment_name=None) + assert "No orchestration deployment found" in str(exc_info.value) + + def test_get_complete_url_uses_deployment_name_from_litellm_params(self): + config = _make_config() + with patch("litellm.module_level_client", _mock_deployments("grounding-orch", "plain-orch")): + url = config.get_complete_url( + api_base=None, + api_key=None, + model="gpt-4o", + optional_params={}, + litellm_params={"deployment_name": "plain-orch"}, + ) + assert url == "https://deploy-plain-orch.sap.com/v2/completion"