fix(vertex_ai): reuse the rerank config so its credential cache survives requests

get_provider_rerank_config built a new VertexAIRerankConfig for every
request, and VertexBase keeps its credential cache on the instance, so
each Vertex rerank loaded and refreshed the Google credentials again
(200-500 ms, synchronous). Return one shared instance from an
lru_cache'd helper, as get_provider_anthropic_messages_config does.

Fixes #47932
This commit is contained in:
Demetre Dzmanashvili 2026-10-22 17:45:29 +02:00
parent f2118fc767
commit e8fb9ed864
2 changed files with 81 additions and 36 deletions

View file

@ -9595,7 +9595,7 @@ class ProviderConfigManager:
return get_nvidia_nim_rerank_config(model)
elif litellm.LlmProviders.VERTEX_AI == provider:
return litellm.VertexAIRerankConfig()
return ProviderConfigManager._get_vertex_ai_rerank_config_cached()
elif litellm.LlmProviders.FIREWORKS_AI == provider:
return litellm.FireworksAIRerankConfig()
elif litellm.LlmProviders.VOYAGE == provider:
@ -9614,6 +9614,16 @@ class ProviderConfigManager:
return get_dashscope_family_rerank_config(provider.value)
return litellm.CohereRerankConfig()
@staticmethod
@lru_cache(maxsize=1)
def _get_vertex_ai_rerank_config_cached() -> BaseRerankConfig:
"""One shared instance, so the `VertexBase` credential cache it holds outlives a request.
A fresh instance starts with an empty `_credentials_project_mapping`, so every
rerank would load and refresh the Google credentials again.
"""
return litellm.VertexAIRerankConfig()
@staticmethod
def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None:
if provider == LlmProviders.PERPLEXITY:

View file

@ -92,13 +92,9 @@ class TestVertexAIRerankTransform:
litellm.vertex_project = None
# Reset mock and set it to raise an error
mock_ensure_access_token.reset_mock()
mock_ensure_access_token.side_effect = ValueError(
"Vertex AI project ID is required"
)
mock_ensure_access_token.side_effect = ValueError("Vertex AI project ID is required")
try:
with pytest.raises(
ValueError, match="Vertex AI project ID is required"
):
with pytest.raises(ValueError, match="Vertex AI project ID is required"):
self.config.get_complete_url(api_base=None, model=self.model)
finally:
litellm.vertex_project = original_project
@ -111,14 +107,10 @@ class TestVertexAIRerankTransform:
importlib.reload(litellm) in conftest.py.
"""
# Mock the authentication at instance level
mock_ensure_access_token = MagicMock(
return_value=("test-access-token", "test-project-123")
)
mock_ensure_access_token = MagicMock(return_value=("test-access-token", "test-project-123"))
self.config._ensure_access_token = mock_ensure_access_token
headers = self.config.validate_environment(
headers={}, model=self.model, api_key=None
)
headers = self.config.validate_environment(headers={}, model=self.model, api_key=None)
expected_headers = {
"Authorization": "Bearer test-access-token",
@ -166,9 +158,7 @@ class TestVertexAIRerankTransform:
"text": "Gemini is a cutting edge large language model created by Google.",
"title": "Custom Title 1",
},
{
"text": "The Gemini zodiac symbol often depicts two figures standing side-by-side."
},
{"text": "The Gemini zodiac symbol often depicts two figures standing side-by-side."},
],
}
@ -178,9 +168,7 @@ class TestVertexAIRerankTransform:
# Verify record structure with custom titles
assert request_data["records"][0]["title"] == "Custom Title 1"
assert (
request_data["records"][1]["title"] == "The Gemini zodiac"
) # First 3 words
assert request_data["records"][1]["title"] == "The Gemini zodiac" # First 3 words
def test_transform_rerank_request_return_documents_mapping(self):
"""Test return_documents to ignoreRecordDetailsInResponse mapping."""
@ -226,9 +214,7 @@ class TestVertexAIRerankTransform:
model=self.model,
optional_rerank_params=optional_params,
headers={},
litellm_params={
"metadata": {"requester_metadata": {"app": "litellm", "tier": "1"}}
},
litellm_params={"metadata": {"requester_metadata": {"app": "litellm", "tier": "1"}}},
)
assert request_data["userLabels"] == {"app": "litellm", "tier": "1"}
@ -243,9 +229,7 @@ class TestVertexAIRerankTransform:
)
# Test missing documents
with pytest.raises(
ValueError, match="documents is required for Vertex AI rerank"
):
with pytest.raises(ValueError, match="documents is required for Vertex AI rerank"):
self.config.transform_rerank_request(
model=self.model,
optional_rerank_params={"query": "test query"},
@ -330,8 +314,7 @@ class TestVertexAIRerankTransform:
def _build_response(self, num_records):
response_data = {
"records": [
{"id": str(i), "score": 1.0 - i / 1000, "title": "t", "content": "c"}
for i in range(num_records)
{"id": str(i), "score": 1.0 - i / 1000, "title": "t", "content": "c"} for i in range(num_records)
]
}
mock_response = MagicMock(spec=httpx.Response)
@ -478,9 +461,7 @@ class TestVertexAIRerankTransform:
# Verify title generation
assert request_data["records"][0]["title"] == "This is a" # First 3 words
assert request_data["records"][1]["title"] == "Short doc" # Less than 3 words
assert (
request_data["records"][2]["title"] == "Another document with"
) # First 3 words
assert request_data["records"][2]["title"] == "Another document with" # First 3 words
def test_record_id_generation(self):
"""Test that record IDs are generated correctly with 0-based indexing."""
@ -561,9 +542,7 @@ class TestVertexAIRerankTransform:
importlib.reload(litellm) in conftest.py.
"""
# Mock the authentication at instance level
mock_ensure_access_token = MagicMock(
return_value=("test-access-token", "test-project-123")
)
mock_ensure_access_token = MagicMock(return_value=("test-access-token", "test-project-123"))
self.config._ensure_access_token = mock_ensure_access_token
optional_params = {
@ -601,9 +580,7 @@ class TestVertexAIRerankTransform:
Uses instance-level mocking to avoid class-reference issues caused by
importlib.reload(litellm) in conftest.py.
"""
mock_ensure_access_token = MagicMock(
return_value=("test-access-token", "project-from-token")
)
mock_ensure_access_token = MagicMock(return_value=("test-access-token", "project-from-token"))
self.config._ensure_access_token = mock_ensure_access_token
optional_params = {
@ -646,3 +623,61 @@ class TestVertexAIRerankTransform:
second_call = mock_ensure_access_token.call_args_list[1]
assert first_call.kwargs["credentials"] == "path/to/credentials.json"
assert second_call.kwargs["credentials"] == "path/to/credentials.json"
def test_provider_config_manager_reuses_vertex_ai_rerank_config_instance():
"""
Regression test: repeated rerank config lookups for Vertex AI return the same
instance, so the credential cache it holds survives across requests.
"""
import litellm
from litellm.utils import ProviderConfigManager
ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()
try:
lookups = [
ProviderConfigManager.get_provider_rerank_config(
model="semantic-ranker-default-004",
provider=litellm.LlmProviders.VERTEX_AI,
api_base=None,
present_version_params=[],
)
for _ in range(2)
]
assert isinstance(lookups[0], VertexAIRerankConfig)
assert lookups[0] is lookups[1]
finally:
ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()
def test_vertex_ai_rerank_loads_credentials_once_across_requests():
"""
Regression test: each rerank request used to build a fresh config, whose empty
credential cache made it load and refresh the Google credentials every time.
"""
import litellm
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.utils import ProviderConfigManager
credentials = MagicMock(expired=False, token="test-token")
ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()
try:
with patch.object(VertexBase, "load_auth", return_value=(credentials, "test-project")) as load_auth:
for _ in range(3):
config = ProviderConfigManager.get_provider_rerank_config(
model="semantic-ranker-default-004",
provider=litellm.LlmProviders.VERTEX_AI,
api_base=None,
present_version_params=[],
)
headers = config.validate_environment(
headers={},
model="semantic-ranker-default-004",
optional_params={"vertex_project": "test-project"},
)
assert headers["Authorization"] == "Bearer test-token"
assert load_auth.call_count == 1
finally:
ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()