From da31c0f410095640198c13b342e48a113bec5940 Mon Sep 17 00:00:00 2001 From: Mrun25 <165297479+Mrun25@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:12:43 +0530 Subject: [PATCH] fix(sap): cache the orchestration deployment URL process-wide --- litellm/llms/sap/chat/transformation.py | 26 +++++- litellm/llms/sap/embed/transformation.py | 24 ++++- .../llms/sap/test_sap_deployment_url_cache.py | 87 +++++++++++++++++++ 3 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 tests/unit/llms/sap/test_sap_deployment_url_cache.py diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index a82dfe72322..89add3b17e2 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -2,10 +2,10 @@ Translate from OpenAI's `/v1/chat/completions` to SAP Generative AI Hub's Orchestration Service`v2/completion` """ +import threading import types from collections.abc import AsyncIterator, Iterator, Mapping -from functools import cached_property -from typing import TYPE_CHECKING, Any, Final, Union +from typing import TYPE_CHECKING, Any, ClassVar, Final, Union import httpx from pydantic import BaseModel @@ -180,8 +180,28 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): self.run_env_setup() return self._resource_group - @cached_property + # Process-wide cache: the provider map builds a fresh config instance per + # call, so an instance-scoped cached_property re-runs the two discovery + # GETs on every LLM call. Keyed on (base_url, resource_group) - the inputs + # that determine the URL - so a credential change never serves a stale URL. + _deployment_url_cache: ClassVar[dict[tuple[str, str], str]] = {} # mutable-ok: process cache, guarded by the lock + _deployment_url_lock: ClassVar[threading.Lock] = threading.Lock() + + @property def deployment_url(self) -> str: + cache_key: Final = (self.base_url, self.resource_group) + cached_url: Final = GenAIHubOrchestrationConfig._deployment_url_cache.get(cache_key) + if cached_url is not None: + return cached_url + with GenAIHubOrchestrationConfig._deployment_url_lock: + rechecked_url: Final = GenAIHubOrchestrationConfig._deployment_url_cache.get(cache_key) + if rechecked_url is not None: + return rechecked_url + fetched_url: Final = self._fetch_deployment_url() + GenAIHubOrchestrationConfig._deployment_url_cache[cache_key] = fetched_url + return fetched_url + + def _fetch_deployment_url(self) -> str: # Keep a short, tight client lifecycle here to avoid fd leaks client: Final = litellm.module_level_client # with httpx.Client(timeout=30) as client: diff --git a/litellm/llms/sap/embed/transformation.py b/litellm/llms/sap/embed/transformation.py index e91dfb91081..caaf4a4c040 100644 --- a/litellm/llms/sap/embed/transformation.py +++ b/litellm/llms/sap/embed/transformation.py @@ -2,8 +2,8 @@ Translates from OpenAI's `/v1/embeddings` to IBM's `/text/embeddings` route. """ -from functools import cached_property -from typing import Final, Literal +import threading +from typing import ClassVar, Final, Literal import httpx from pydantic import Field @@ -97,8 +97,26 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig): } return headers - @cached_property + # Process-wide cache, same reasoning as the chat config: per-call config + # instances would otherwise re-run the two discovery GETs on every call. + _deployment_url_cache: ClassVar[dict[tuple[str, str], str]] = {} # mutable-ok: process cache, guarded by the lock + _deployment_url_lock: ClassVar[threading.Lock] = threading.Lock() + + @property def deployment_url(self) -> str: + cache_key: Final = (self.base_url, self.resource_group) + cached_url: Final = GenAIHubEmbeddingConfig._deployment_url_cache.get(cache_key) + if cached_url is not None: + return cached_url + with GenAIHubEmbeddingConfig._deployment_url_lock: + rechecked_url: Final = GenAIHubEmbeddingConfig._deployment_url_cache.get(cache_key) + if rechecked_url is not None: + return rechecked_url + fetched_url: Final = self._fetch_deployment_url() + GenAIHubEmbeddingConfig._deployment_url_cache[cache_key] = fetched_url + return fetched_url + + def _fetch_deployment_url(self) -> str: with httpx.Client(timeout=30) as client: valid_deployments: Final = [] deployments: Final = client.get(self.base_url + "/lm/deployments", headers=self.headers).json() diff --git a/tests/unit/llms/sap/test_sap_deployment_url_cache.py b/tests/unit/llms/sap/test_sap_deployment_url_cache.py new file mode 100644 index 00000000000..aef2738dd07 --- /dev/null +++ b/tests/unit/llms/sap/test_sap_deployment_url_cache.py @@ -0,0 +1,87 @@ +"""Process-wide deployment_url caching for SAP GenAI Hub configs (issue #47096). + +The provider map builds a fresh config instance on every LLM call, so an +instance-scoped cached_property re-ran the two orchestration-discovery GETs on +every call. The cache now lives on the class, keyed on (base_url, resource_group). +""" + +import pytest + +from litellm.llms.sap.chat.transformation import GenAIHubOrchestrationConfig +from litellm.llms.sap.embed.transformation import GenAIHubEmbeddingConfig + + +@pytest.fixture(autouse=True) +def clear_deployment_url_caches(): + GenAIHubOrchestrationConfig._deployment_url_cache.clear() + GenAIHubEmbeddingConfig._deployment_url_cache.clear() + yield + GenAIHubOrchestrationConfig._deployment_url_cache.clear() + GenAIHubEmbeddingConfig._deployment_url_cache.clear() + + +def _chat_config(base_url: str, resource_group: str) -> GenAIHubOrchestrationConfig: + config = GenAIHubOrchestrationConfig() + config._base_url = base_url + config._resource_group = resource_group + return config + + +def _embed_config(monkeypatch, base_url: str, resource_group: str) -> GenAIHubEmbeddingConfig: + monkeypatch.setattr( + "litellm.llms.sap.embed.transformation.get_token_creator", + lambda service_key=None: (lambda: "token", base_url, resource_group), + ) + return GenAIHubEmbeddingConfig() + + +def test_chat_deployment_url_fetched_once_across_instances(monkeypatch): + fetches = [] + + def fake_fetch(self): + fetches.append(self) + return "https://deploy.example/v2" + + monkeypatch.setattr(GenAIHubOrchestrationConfig, "_fetch_deployment_url", fake_fetch) + + # Two distinct instances, like the per-call lambda in the provider map + first = _chat_config("https://api.example", "default") + second = _chat_config("https://api.example", "default") + + assert first.deployment_url == "https://deploy.example/v2" + assert second.deployment_url == "https://deploy.example/v2" + assert len(fetches) == 1 + + +def test_chat_deployment_url_cache_keyed_on_base_url_and_resource_group(monkeypatch): + fetches = [] + + def fake_fetch(self): + fetches.append(self) + return f"https://deploy.example/{self.resource_group}" + + monkeypatch.setattr(GenAIHubOrchestrationConfig, "_fetch_deployment_url", fake_fetch) + + default_rg = _chat_config("https://api.example", "default") + other_rg = _chat_config("https://api.example", "other") + + assert default_rg.deployment_url == "https://deploy.example/default" + assert other_rg.deployment_url == "https://deploy.example/other" + assert len(fetches) == 2 + + +def test_embed_deployment_url_fetched_once_across_instances(monkeypatch): + fetches = [] + + def fake_fetch(self): + fetches.append(self) + return "https://deploy.example/v2" + + monkeypatch.setattr(GenAIHubEmbeddingConfig, "_fetch_deployment_url", fake_fetch) + + first = _embed_config(monkeypatch, "https://api.example", "default") + second = _embed_config(monkeypatch, "https://api.example", "default") + + assert first.deployment_url == "https://deploy.example/v2" + assert second.deployment_url == "https://deploy.example/v2" + assert len(fetches) == 1