fix(sap): cache the orchestration deployment URL process-wide

litellm/llms/sap/chat/transformation.py

@@ -2,10 +2,10 @@
 Translate from OpenAI's `/v1/chat/completions` to SAP Generative AI Hub's Orchestration Service`v2/completion`
 """
 
+import threading
 import types
 from collections.abc import AsyncIterator, Iterator, Mapping
-from functools import cached_property
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Union
 
 import httpx
 from pydantic import BaseModel
@@ -180,8 +180,28 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
             self.run_env_setup()
         return self._resource_group
 
-    @cached_property
+    # Process-wide cache: the provider map builds a fresh config instance per
+    # call, so an instance-scoped cached_property re-runs the two discovery
+    # GETs on every LLM call. Keyed on (base_url, resource_group) - the inputs
+    # that determine the URL - so a credential change never serves a stale URL.
+    _deployment_url_cache: ClassVar[dict[tuple[str, str], str]] = {}  # mutable-ok: process cache, guarded by the lock
+    _deployment_url_lock: ClassVar[threading.Lock] = threading.Lock()
+
+    @property
     def deployment_url(self) -> str:
+        cache_key: Final = (self.base_url, self.resource_group)
+        cached_url: Final = GenAIHubOrchestrationConfig._deployment_url_cache.get(cache_key)
+        if cached_url is not None:
+            return cached_url
+        with GenAIHubOrchestrationConfig._deployment_url_lock:
+            rechecked_url: Final = GenAIHubOrchestrationConfig._deployment_url_cache.get(cache_key)
+            if rechecked_url is not None:
+                return rechecked_url
+            fetched_url: Final = self._fetch_deployment_url()
+            GenAIHubOrchestrationConfig._deployment_url_cache[cache_key] = fetched_url
+            return fetched_url
+
+    def _fetch_deployment_url(self) -> str:
         # Keep a short, tight client lifecycle here to avoid fd leaks
         client: Final = litellm.module_level_client
         # with httpx.Client(timeout=30) as client:

litellm/llms/sap/embed/transformation.py

@@ -2,8 +2,8 @@
 Translates from OpenAI's `/v1/embeddings` to IBM's `/text/embeddings` route.
 """
 
-from functools import cached_property
-from typing import Final, Literal
+import threading
+from typing import ClassVar, Final, Literal
 
 import httpx
 from pydantic import Field
@@ -97,8 +97,26 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig):
         }
         return headers
 
-    @cached_property
+    # Process-wide cache, same reasoning as the chat config: per-call config
+    # instances would otherwise re-run the two discovery GETs on every call.
+    _deployment_url_cache: ClassVar[dict[tuple[str, str], str]] = {}  # mutable-ok: process cache, guarded by the lock
+    _deployment_url_lock: ClassVar[threading.Lock] = threading.Lock()
+
+    @property
     def deployment_url(self) -> str:
+        cache_key: Final = (self.base_url, self.resource_group)
+        cached_url: Final = GenAIHubEmbeddingConfig._deployment_url_cache.get(cache_key)
+        if cached_url is not None:
+            return cached_url
+        with GenAIHubEmbeddingConfig._deployment_url_lock:
+            rechecked_url: Final = GenAIHubEmbeddingConfig._deployment_url_cache.get(cache_key)
+            if rechecked_url is not None:
+                return rechecked_url
+            fetched_url: Final = self._fetch_deployment_url()
+            GenAIHubEmbeddingConfig._deployment_url_cache[cache_key] = fetched_url
+            return fetched_url
+
+    def _fetch_deployment_url(self) -> str:
         with httpx.Client(timeout=30) as client:
             valid_deployments: Final = []
             deployments: Final = client.get(self.base_url + "/lm/deployments", headers=self.headers).json()

tests/unit/llms/sap/test_sap_deployment_url_cache.py

@@ -0,0 +1,87 @@
+"""Process-wide deployment_url caching for SAP GenAI Hub configs (issue #47096).
+
+The provider map builds a fresh config instance on every LLM call, so an
+instance-scoped cached_property re-ran the two orchestration-discovery GETs on
+every call. The cache now lives on the class, keyed on (base_url, resource_group).
+"""
+
+import pytest
+
+from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig
+from litellm.llms.sap.embed.transformation import GenAIHubEmbeddingConfig
+
+
+@pytest.fixture(autouse=True)
+def clear_deployment_url_caches():
+    GenAIHubOrchestrationConfig._deployment_url_cache.clear()
+    GenAIHubEmbeddingConfig._deployment_url_cache.clear()
+    yield
+    GenAIHubOrchestrationConfig._deployment_url_cache.clear()
+    GenAIHubEmbeddingConfig._deployment_url_cache.clear()
+
+
+def _chat_config(base_url: str, resource_group: str) -> GenAIHubOrchestrationConfig:
+    config = GenAIHubOrchestrationConfig()
+    config._base_url = base_url
+    config._resource_group = resource_group
+    return config
+
+
+def _embed_config(monkeypatch, base_url: str, resource_group: str) -> GenAIHubEmbeddingConfig:
+    monkeypatch.setattr(
+        "litellm.llms.sap.embed.transformation.get_token_creator",
+        lambda service_key=None: (lambda: "token", base_url, resource_group),
+    )
+    return GenAIHubEmbeddingConfig()
+
+
+def test_chat_deployment_url_fetched_once_across_instances(monkeypatch):
+    fetches = []
+
+    def fake_fetch(self):
+        fetches.append(self)
+        return "https://deploy.example/v2"
+
+    monkeypatch.setattr(GenAIHubOrchestrationConfig, "_fetch_deployment_url", fake_fetch)
+
+    # Two distinct instances, like the per-call lambda in the provider map
+    first = _chat_config("https://api.example", "default")
+    second = _chat_config("https://api.example", "default")
+
+    assert first.deployment_url == "https://deploy.example/v2"
+    assert second.deployment_url == "https://deploy.example/v2"
+    assert len(fetches) == 1
+
+
+def test_chat_deployment_url_cache_keyed_on_base_url_and_resource_group(monkeypatch):
+    fetches = []
+
+    def fake_fetch(self):
+        fetches.append(self)
+        return f"https://deploy.example/{self.resource_group}"
+
+    monkeypatch.setattr(GenAIHubOrchestrationConfig, "_fetch_deployment_url", fake_fetch)
+
+    default_rg = _chat_config("https://api.example", "default")
+    other_rg = _chat_config("https://api.example", "other")
+
+    assert default_rg.deployment_url == "https://deploy.example/default"
+    assert other_rg.deployment_url == "https://deploy.example/other"
+    assert len(fetches) == 2
+
+
+def test_embed_deployment_url_fetched_once_across_instances(monkeypatch):
+    fetches = []
+
+    def fake_fetch(self):
+        fetches.append(self)
+        return "https://deploy.example/v2"
+
+    monkeypatch.setattr(GenAIHubEmbeddingConfig, "_fetch_deployment_url", fake_fetch)
+
+    first = _embed_config(monkeypatch, "https://api.example", "default")
+    second = _embed_config(monkeypatch, "https://api.example", "default")
+
+    assert first.deployment_url == "https://deploy.example/v2"
+    assert second.deployment_url == "https://deploy.example/v2"
+    assert len(fetches) == 1