mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(vertex_ai): reuse the rerank config so its credential cache survives requests
get_provider_rerank_config built a new VertexAIRerankConfig for every request, and VertexBase keeps its credential cache on the instance, so each Vertex rerank loaded and refreshed the Google credentials again (200-500 ms, synchronous). Return one shared instance from an lru_cache'd helper, as get_provider_anthropic_messages_config does. Fixes #47932
This commit is contained in:
parent
f2118fc767
commit
e8fb9ed864
2 changed files with 81 additions and 36 deletions
|
|
@ -9595,7 +9595,7 @@ class ProviderConfigManager:
|
|||
|
||||
return get_nvidia_nim_rerank_config(model)
|
||||
elif litellm.LlmProviders.VERTEX_AI == provider:
|
||||
return litellm.VertexAIRerankConfig()
|
||||
return ProviderConfigManager._get_vertex_ai_rerank_config_cached()
|
||||
elif litellm.LlmProviders.FIREWORKS_AI == provider:
|
||||
return litellm.FireworksAIRerankConfig()
|
||||
elif litellm.LlmProviders.VOYAGE == provider:
|
||||
|
|
@ -9614,6 +9614,16 @@ class ProviderConfigManager:
|
|||
return get_dashscope_family_rerank_config(provider.value)
|
||||
return litellm.CohereRerankConfig()
|
||||
|
||||
@staticmethod
|
||||
@lru_cache(maxsize=1)
|
||||
def _get_vertex_ai_rerank_config_cached() -> BaseRerankConfig:
|
||||
"""One shared instance, so the `VertexBase` credential cache it holds outlives a request.
|
||||
|
||||
A fresh instance starts with an empty `_credentials_project_mapping`, so every
|
||||
rerank would load and refresh the Google credentials again.
|
||||
"""
|
||||
return litellm.VertexAIRerankConfig()
|
||||
|
||||
@staticmethod
|
||||
def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None:
|
||||
if provider == LlmProviders.PERPLEXITY:
|
||||
|
|
|
|||
|
|
@ -92,13 +92,9 @@ class TestVertexAIRerankTransform:
|
|||
litellm.vertex_project = None
|
||||
# Reset mock and set it to raise an error
|
||||
mock_ensure_access_token.reset_mock()
|
||||
mock_ensure_access_token.side_effect = ValueError(
|
||||
"Vertex AI project ID is required"
|
||||
)
|
||||
mock_ensure_access_token.side_effect = ValueError("Vertex AI project ID is required")
|
||||
try:
|
||||
with pytest.raises(
|
||||
ValueError, match="Vertex AI project ID is required"
|
||||
):
|
||||
with pytest.raises(ValueError, match="Vertex AI project ID is required"):
|
||||
self.config.get_complete_url(api_base=None, model=self.model)
|
||||
finally:
|
||||
litellm.vertex_project = original_project
|
||||
|
|
@ -111,14 +107,10 @@ class TestVertexAIRerankTransform:
|
|||
importlib.reload(litellm) in conftest.py.
|
||||
"""
|
||||
# Mock the authentication at instance level
|
||||
mock_ensure_access_token = MagicMock(
|
||||
return_value=("test-access-token", "test-project-123")
|
||||
)
|
||||
mock_ensure_access_token = MagicMock(return_value=("test-access-token", "test-project-123"))
|
||||
self.config._ensure_access_token = mock_ensure_access_token
|
||||
|
||||
headers = self.config.validate_environment(
|
||||
headers={}, model=self.model, api_key=None
|
||||
)
|
||||
headers = self.config.validate_environment(headers={}, model=self.model, api_key=None)
|
||||
|
||||
expected_headers = {
|
||||
"Authorization": "Bearer test-access-token",
|
||||
|
|
@ -166,9 +158,7 @@ class TestVertexAIRerankTransform:
|
|||
"text": "Gemini is a cutting edge large language model created by Google.",
|
||||
"title": "Custom Title 1",
|
||||
},
|
||||
{
|
||||
"text": "The Gemini zodiac symbol often depicts two figures standing side-by-side."
|
||||
},
|
||||
{"text": "The Gemini zodiac symbol often depicts two figures standing side-by-side."},
|
||||
],
|
||||
}
|
||||
|
||||
|
|
@ -178,9 +168,7 @@ class TestVertexAIRerankTransform:
|
|||
|
||||
# Verify record structure with custom titles
|
||||
assert request_data["records"][0]["title"] == "Custom Title 1"
|
||||
assert (
|
||||
request_data["records"][1]["title"] == "The Gemini zodiac"
|
||||
) # First 3 words
|
||||
assert request_data["records"][1]["title"] == "The Gemini zodiac" # First 3 words
|
||||
|
||||
def test_transform_rerank_request_return_documents_mapping(self):
|
||||
"""Test return_documents to ignoreRecordDetailsInResponse mapping."""
|
||||
|
|
@ -226,9 +214,7 @@ class TestVertexAIRerankTransform:
|
|||
model=self.model,
|
||||
optional_rerank_params=optional_params,
|
||||
headers={},
|
||||
litellm_params={
|
||||
"metadata": {"requester_metadata": {"app": "litellm", "tier": "1"}}
|
||||
},
|
||||
litellm_params={"metadata": {"requester_metadata": {"app": "litellm", "tier": "1"}}},
|
||||
)
|
||||
assert request_data["userLabels"] == {"app": "litellm", "tier": "1"}
|
||||
|
||||
|
|
@ -243,9 +229,7 @@ class TestVertexAIRerankTransform:
|
|||
)
|
||||
|
||||
# Test missing documents
|
||||
with pytest.raises(
|
||||
ValueError, match="documents is required for Vertex AI rerank"
|
||||
):
|
||||
with pytest.raises(ValueError, match="documents is required for Vertex AI rerank"):
|
||||
self.config.transform_rerank_request(
|
||||
model=self.model,
|
||||
optional_rerank_params={"query": "test query"},
|
||||
|
|
@ -330,8 +314,7 @@ class TestVertexAIRerankTransform:
|
|||
def _build_response(self, num_records):
|
||||
response_data = {
|
||||
"records": [
|
||||
{"id": str(i), "score": 1.0 - i / 1000, "title": "t", "content": "c"}
|
||||
for i in range(num_records)
|
||||
{"id": str(i), "score": 1.0 - i / 1000, "title": "t", "content": "c"} for i in range(num_records)
|
||||
]
|
||||
}
|
||||
mock_response = MagicMock(spec=httpx.Response)
|
||||
|
|
@ -478,9 +461,7 @@ class TestVertexAIRerankTransform:
|
|||
# Verify title generation
|
||||
assert request_data["records"][0]["title"] == "This is a" # First 3 words
|
||||
assert request_data["records"][1]["title"] == "Short doc" # Less than 3 words
|
||||
assert (
|
||||
request_data["records"][2]["title"] == "Another document with"
|
||||
) # First 3 words
|
||||
assert request_data["records"][2]["title"] == "Another document with" # First 3 words
|
||||
|
||||
def test_record_id_generation(self):
|
||||
"""Test that record IDs are generated correctly with 0-based indexing."""
|
||||
|
|
@ -561,9 +542,7 @@ class TestVertexAIRerankTransform:
|
|||
importlib.reload(litellm) in conftest.py.
|
||||
"""
|
||||
# Mock the authentication at instance level
|
||||
mock_ensure_access_token = MagicMock(
|
||||
return_value=("test-access-token", "test-project-123")
|
||||
)
|
||||
mock_ensure_access_token = MagicMock(return_value=("test-access-token", "test-project-123"))
|
||||
self.config._ensure_access_token = mock_ensure_access_token
|
||||
|
||||
optional_params = {
|
||||
|
|
@ -601,9 +580,7 @@ class TestVertexAIRerankTransform:
|
|||
Uses instance-level mocking to avoid class-reference issues caused by
|
||||
importlib.reload(litellm) in conftest.py.
|
||||
"""
|
||||
mock_ensure_access_token = MagicMock(
|
||||
return_value=("test-access-token", "project-from-token")
|
||||
)
|
||||
mock_ensure_access_token = MagicMock(return_value=("test-access-token", "project-from-token"))
|
||||
self.config._ensure_access_token = mock_ensure_access_token
|
||||
|
||||
optional_params = {
|
||||
|
|
@ -646,3 +623,61 @@ class TestVertexAIRerankTransform:
|
|||
second_call = mock_ensure_access_token.call_args_list[1]
|
||||
assert first_call.kwargs["credentials"] == "path/to/credentials.json"
|
||||
assert second_call.kwargs["credentials"] == "path/to/credentials.json"
|
||||
|
||||
|
||||
def test_provider_config_manager_reuses_vertex_ai_rerank_config_instance():
|
||||
"""
|
||||
Regression test: repeated rerank config lookups for Vertex AI return the same
|
||||
instance, so the credential cache it holds survives across requests.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()
|
||||
try:
|
||||
lookups = [
|
||||
ProviderConfigManager.get_provider_rerank_config(
|
||||
model="semantic-ranker-default-004",
|
||||
provider=litellm.LlmProviders.VERTEX_AI,
|
||||
api_base=None,
|
||||
present_version_params=[],
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
|
||||
assert isinstance(lookups[0], VertexAIRerankConfig)
|
||||
assert lookups[0] is lookups[1]
|
||||
finally:
|
||||
ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()
|
||||
|
||||
|
||||
def test_vertex_ai_rerank_loads_credentials_once_across_requests():
|
||||
"""
|
||||
Regression test: each rerank request used to build a fresh config, whose empty
|
||||
credential cache made it load and refresh the Google credentials every time.
|
||||
"""
|
||||
import litellm
|
||||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
credentials = MagicMock(expired=False, token="test-token")
|
||||
ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()
|
||||
try:
|
||||
with patch.object(VertexBase, "load_auth", return_value=(credentials, "test-project")) as load_auth:
|
||||
for _ in range(3):
|
||||
config = ProviderConfigManager.get_provider_rerank_config(
|
||||
model="semantic-ranker-default-004",
|
||||
provider=litellm.LlmProviders.VERTEX_AI,
|
||||
api_base=None,
|
||||
present_version_params=[],
|
||||
)
|
||||
headers = config.validate_environment(
|
||||
headers={},
|
||||
model="semantic-ranker-default-004",
|
||||
optional_params={"vertex_project": "test-project"},
|
||||
)
|
||||
assert headers["Authorization"] == "Bearer test-token"
|
||||
|
||||
assert load_auth.call_count == 1
|
||||
finally:
|
||||
ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue