fix(sap): cache the orchestration deployment URL process-wide

Cherry-picked from the PR: https://github.com/BerriAI/litellm/pulls/47102

litellm/llms/sap/chat/transformation.py

@@ -2,10 +2,11 @@
 Translate from OpenAI's `/v1/chat/completions` to SAP Generative AI Hub's Orchestration Service`v2/completion`
 """
 
+import threading
 import types
 import os
 from collections.abc import AsyncIterator, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Final, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Union
 
 import httpx
 from pydantic import BaseModel
@@ -187,6 +188,13 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
             self.run_env_setup()
         return self._resource_group
 
+    # Process-wide cache: the provider map builds a fresh config instance per
+    # call, so an instance-scoped cached_property re-runs the two discovery
+    # GETs on every LLM call. Keyed on (base_url, resource_group) - the inputs
+    # that determine the URL - so a credential change never serves a stale URL.
+    _deployment_url_cache: ClassVar[dict[tuple[str, str], str]] = {}  # mutable-ok: process cache, guarded by the lock
+    _deployment_url_lock: ClassVar[threading.Lock] = threading.Lock()
+
     @property
     def deployment_url(self) -> str:
         """Resolve the orchestration deployment URL, with caching.
@@ -200,12 +208,17 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
         handled one level up in ``get_complete_url`` before this property
         is ever called.
         """
-        cached = self._cached_deployment_url
-        if cached is None:
-            env = os.environ.get(_AICORE_ORCHESTRATION_DEPLOYMENT_URL_ENV_VAR)
-            cached = env or self._fetch_deployment_url()
-            self._cached_deployment_url = cached
-        return cached
+        cache_key: Final = (self.base_url, self.resource_group)
+        cached_url: Final = GenAIHubOrchestrationConfig._deployment_url_cache.get(cache_key)
+        if cached_url is not None:
+            return cached_url
+        with GenAIHubOrchestrationConfig._deployment_url_lock:
+            rechecked_url: Final = GenAIHubOrchestrationConfig._deployment_url_cache.get(cache_key)
+            if rechecked_url is not None:
+                return rechecked_url
+            fetched_url: Final = os.environ.get(_AICORE_ORCHESTRATION_DEPLOYMENT_URL_ENV_VAR) or self._fetch_deployment_url()
+            GenAIHubOrchestrationConfig._deployment_url_cache[cache_key] = fetched_url
+            return fetched_url
 
     def _fetch_deployment_url(self) -> str:
         client: Final = litellm.module_level_client

litellm/llms/sap/embed/transformation.py

@@ -2,8 +2,8 @@
 Translates from OpenAI's `/v1/embeddings` to IBM's `/text/embeddings` route.
 """
 
-from functools import cached_property
-from typing import Final, Literal
+import threading
+from typing import ClassVar, Final, Literal
 
 import httpx
 from pydantic import Field
@@ -98,8 +98,26 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig):
         }
         return headers
 
-    @cached_property
+    # Process-wide cache, same reasoning as the chat config: per-call config
+    # instances would otherwise re-run the two discovery GETs on every call.
+    _deployment_url_cache: ClassVar[dict[tuple[str, str], str]] = {}  # mutable-ok: process cache, guarded by the lock
+    _deployment_url_lock: ClassVar[threading.Lock] = threading.Lock()
+
+    @property
     def deployment_url(self) -> str:
+        cache_key: Final = (self.base_url, self.resource_group)
+        cached_url: Final = GenAIHubEmbeddingConfig._deployment_url_cache.get(cache_key)
+        if cached_url is not None:
+            return cached_url
+        with GenAIHubEmbeddingConfig._deployment_url_lock:
+            rechecked_url: Final = GenAIHubEmbeddingConfig._deployment_url_cache.get(cache_key)
+            if rechecked_url is not None:
+                return rechecked_url
+            fetched_url: Final = self._fetch_deployment_url()
+            GenAIHubEmbeddingConfig._deployment_url_cache[cache_key] = fetched_url
+            return fetched_url
+
+    def _fetch_deployment_url(self) -> str:
         with httpx.Client(timeout=30) as client:
             valid_deployments: Final = []
             deployments: Final[AICoreDeploymentList] = client.get(