From e8fb9ed864f8441b36085fc2e251bedef51697cb Mon Sep 17 00:00:00 2001 From: Demetre Dzmanashvili Date: Thu, 22 Oct 2026 17:45:29 +0200 Subject: [PATCH] fix(vertex_ai): reuse the rerank config so its credential cache survives requests get_provider_rerank_config built a new VertexAIRerankConfig for every request, and VertexBase keeps its credential cache on the instance, so each Vertex rerank loaded and refreshed the Google credentials again (200-500 ms, synchronous). Return one shared instance from an lru_cache'd helper, as get_provider_anthropic_messages_config does. Fixes #47932 --- litellm/utils.py | 12 +- .../test_vertex_ai_rerank_transformation.py | 105 ++++++++++++------ 2 files changed, 81 insertions(+), 36 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 7de7ce6188d..3489dae9187 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9595,7 +9595,7 @@ class ProviderConfigManager: return get_nvidia_nim_rerank_config(model) elif litellm.LlmProviders.VERTEX_AI == provider: - return litellm.VertexAIRerankConfig() + return ProviderConfigManager._get_vertex_ai_rerank_config_cached() elif litellm.LlmProviders.FIREWORKS_AI == provider: return litellm.FireworksAIRerankConfig() elif litellm.LlmProviders.VOYAGE == provider: @@ -9614,6 +9614,16 @@ class ProviderConfigManager: return get_dashscope_family_rerank_config(provider.value) return litellm.CohereRerankConfig() + @staticmethod + @lru_cache(maxsize=1) + def _get_vertex_ai_rerank_config_cached() -> BaseRerankConfig: + """One shared instance, so the `VertexBase` credential cache it holds outlives a request. + + A fresh instance starts with an empty `_credentials_project_mapping`, so every + rerank would load and refresh the Google credentials again. + """ + return litellm.VertexAIRerankConfig() + @staticmethod def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None: if provider == LlmProviders.PERPLEXITY: diff --git a/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py index 630b2e1eb34..6fa9e7ff89f 100644 --- a/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py +++ b/tests/unit/llms/vertex_ai/rerank/test_vertex_ai_rerank_transformation.py @@ -92,13 +92,9 @@ class TestVertexAIRerankTransform: litellm.vertex_project = None # Reset mock and set it to raise an error mock_ensure_access_token.reset_mock() - mock_ensure_access_token.side_effect = ValueError( - "Vertex AI project ID is required" - ) + mock_ensure_access_token.side_effect = ValueError("Vertex AI project ID is required") try: - with pytest.raises( - ValueError, match="Vertex AI project ID is required" - ): + with pytest.raises(ValueError, match="Vertex AI project ID is required"): self.config.get_complete_url(api_base=None, model=self.model) finally: litellm.vertex_project = original_project @@ -111,14 +107,10 @@ class TestVertexAIRerankTransform: importlib.reload(litellm) in conftest.py. """ # Mock the authentication at instance level - mock_ensure_access_token = MagicMock( - return_value=("test-access-token", "test-project-123") - ) + mock_ensure_access_token = MagicMock(return_value=("test-access-token", "test-project-123")) self.config._ensure_access_token = mock_ensure_access_token - headers = self.config.validate_environment( - headers={}, model=self.model, api_key=None - ) + headers = self.config.validate_environment(headers={}, model=self.model, api_key=None) expected_headers = { "Authorization": "Bearer test-access-token", @@ -166,9 +158,7 @@ class TestVertexAIRerankTransform: "text": "Gemini is a cutting edge large language model created by Google.", "title": "Custom Title 1", }, - { - "text": "The Gemini zodiac symbol often depicts two figures standing side-by-side." - }, + {"text": "The Gemini zodiac symbol often depicts two figures standing side-by-side."}, ], } @@ -178,9 +168,7 @@ class TestVertexAIRerankTransform: # Verify record structure with custom titles assert request_data["records"][0]["title"] == "Custom Title 1" - assert ( - request_data["records"][1]["title"] == "The Gemini zodiac" - ) # First 3 words + assert request_data["records"][1]["title"] == "The Gemini zodiac" # First 3 words def test_transform_rerank_request_return_documents_mapping(self): """Test return_documents to ignoreRecordDetailsInResponse mapping.""" @@ -226,9 +214,7 @@ class TestVertexAIRerankTransform: model=self.model, optional_rerank_params=optional_params, headers={}, - litellm_params={ - "metadata": {"requester_metadata": {"app": "litellm", "tier": "1"}} - }, + litellm_params={"metadata": {"requester_metadata": {"app": "litellm", "tier": "1"}}}, ) assert request_data["userLabels"] == {"app": "litellm", "tier": "1"} @@ -243,9 +229,7 @@ class TestVertexAIRerankTransform: ) # Test missing documents - with pytest.raises( - ValueError, match="documents is required for Vertex AI rerank" - ): + with pytest.raises(ValueError, match="documents is required for Vertex AI rerank"): self.config.transform_rerank_request( model=self.model, optional_rerank_params={"query": "test query"}, @@ -330,8 +314,7 @@ class TestVertexAIRerankTransform: def _build_response(self, num_records): response_data = { "records": [ - {"id": str(i), "score": 1.0 - i / 1000, "title": "t", "content": "c"} - for i in range(num_records) + {"id": str(i), "score": 1.0 - i / 1000, "title": "t", "content": "c"} for i in range(num_records) ] } mock_response = MagicMock(spec=httpx.Response) @@ -478,9 +461,7 @@ class TestVertexAIRerankTransform: # Verify title generation assert request_data["records"][0]["title"] == "This is a" # First 3 words assert request_data["records"][1]["title"] == "Short doc" # Less than 3 words - assert ( - request_data["records"][2]["title"] == "Another document with" - ) # First 3 words + assert request_data["records"][2]["title"] == "Another document with" # First 3 words def test_record_id_generation(self): """Test that record IDs are generated correctly with 0-based indexing.""" @@ -561,9 +542,7 @@ class TestVertexAIRerankTransform: importlib.reload(litellm) in conftest.py. """ # Mock the authentication at instance level - mock_ensure_access_token = MagicMock( - return_value=("test-access-token", "test-project-123") - ) + mock_ensure_access_token = MagicMock(return_value=("test-access-token", "test-project-123")) self.config._ensure_access_token = mock_ensure_access_token optional_params = { @@ -601,9 +580,7 @@ class TestVertexAIRerankTransform: Uses instance-level mocking to avoid class-reference issues caused by importlib.reload(litellm) in conftest.py. """ - mock_ensure_access_token = MagicMock( - return_value=("test-access-token", "project-from-token") - ) + mock_ensure_access_token = MagicMock(return_value=("test-access-token", "project-from-token")) self.config._ensure_access_token = mock_ensure_access_token optional_params = { @@ -646,3 +623,61 @@ class TestVertexAIRerankTransform: second_call = mock_ensure_access_token.call_args_list[1] assert first_call.kwargs["credentials"] == "path/to/credentials.json" assert second_call.kwargs["credentials"] == "path/to/credentials.json" + + +def test_provider_config_manager_reuses_vertex_ai_rerank_config_instance(): + """ + Regression test: repeated rerank config lookups for Vertex AI return the same + instance, so the credential cache it holds survives across requests. + """ + import litellm + from litellm.utils import ProviderConfigManager + + ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear() + try: + lookups = [ + ProviderConfigManager.get_provider_rerank_config( + model="semantic-ranker-default-004", + provider=litellm.LlmProviders.VERTEX_AI, + api_base=None, + present_version_params=[], + ) + for _ in range(2) + ] + + assert isinstance(lookups[0], VertexAIRerankConfig) + assert lookups[0] is lookups[1] + finally: + ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear() + + +def test_vertex_ai_rerank_loads_credentials_once_across_requests(): + """ + Regression test: each rerank request used to build a fresh config, whose empty + credential cache made it load and refresh the Google credentials every time. + """ + import litellm + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + from litellm.utils import ProviderConfigManager + + credentials = MagicMock(expired=False, token="test-token") + ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear() + try: + with patch.object(VertexBase, "load_auth", return_value=(credentials, "test-project")) as load_auth: + for _ in range(3): + config = ProviderConfigManager.get_provider_rerank_config( + model="semantic-ranker-default-004", + provider=litellm.LlmProviders.VERTEX_AI, + api_base=None, + present_version_params=[], + ) + headers = config.validate_environment( + headers={}, + model="semantic-ranker-default-004", + optional_params={"vertex_project": "test-project"}, + ) + assert headers["Authorization"] == "Bearer test-token" + + assert load_auth.call_count == 1 + finally: + ProviderConfigManager._get_vertex_ai_rerank_config_cached.cache_clear()