From 4235a5d5142b636d96b7b59df70ad9a008e720aa Mon Sep 17 00:00:00 2001 From: Matthew Lapointe Date: Sun, 19 Apr 2026 12:33:13 -0400 Subject: [PATCH] fix: reuse cached credentials in VertexAIPartnerModels instead of creating new VertexLLM per request VertexAIPartnerModels.completion() was creating a throwaway VertexLLM() instance on every call to get an access token, bypassing the credential cache inherited from VertexBase. This caused a fresh token fetch for every single request, adding significant latency overhead. Fix: call super().__init__() to initialize VertexBase's credential cache, and use self._ensure_access_token() instead of a new VertexLLM instance. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../vertex_ai_partner_models/main.py | 9 +- .../test_partner_models_credential_reuse.py | 122 ++++++++++++++++++ 2 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index 123d925f7c1..839c71efa51 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -45,7 +45,7 @@ class PartnerModelPrefixes(str, Enum): class VertexAIPartnerModels(VertexBase): def __init__(self) -> None: - pass + super().__init__() @staticmethod def is_vertex_partner_model(model: str): @@ -116,9 +116,6 @@ class VertexAIPartnerModels(VertexBase): CodestralTextCompletion, ) from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexLLM, - ) except Exception as e: raise VertexAIError( status_code=400, @@ -133,9 +130,7 @@ class VertexAIPartnerModels(VertexBase): message="""Upgrade vertex ai. Run `pip install "google-cloud-aiplatform>=1.38"`""", ) try: - vertex_httpx_logic = VertexLLM() - - access_token, project_id = vertex_httpx_logic._ensure_access_token( + access_token, project_id = self._ensure_access_token( credentials=vertex_credentials, project_id=vertex_project, custom_llm_provider="vertex_ai", diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py new file mode 100644 index 00000000000..4f9e784b76d --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/test_partner_models_credential_reuse.py @@ -0,0 +1,122 @@ +""" +Test that VertexAIPartnerModels reuses cached credentials from VertexBase +instead of creating a new VertexLLM instance on every request. +""" + +import sys +from unittest.mock import MagicMock, patch + +from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( + VertexAIPartnerModels, +) + + +class TestPartnerModelsCredentialReuse: + def test_init_calls_super(self): + """VertexAIPartnerModels.__init__ must call super().__init__() so that + the VertexBase credential cache is initialized.""" + partner = VertexAIPartnerModels() + # These attributes are set by VertexBase.__init__ + assert hasattr(partner, "_credentials_project_mapping") + assert isinstance(partner._credentials_project_mapping, dict) + assert hasattr(partner, "access_token") + assert hasattr(partner, "project_id") + + def test_completion_uses_self_ensure_access_token(self): + """completion() should call self._ensure_access_token, not create a + throwaway VertexLLM instance. This ensures the credential cache on the + singleton is reused across calls.""" + partner = VertexAIPartnerModels() + + # Mock vertexai import and the completion handler + mock_vertexai = MagicMock() + mock_vertexai.preview = MagicMock() + mock_vertexai.preview.language_models = MagicMock() + + with ( + patch.dict(sys.modules, {"vertexai": mock_vertexai}), + patch.object( + partner, + "_ensure_access_token", + return_value=("cached-token", "test-project"), + ) as mock_ensure, + patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.main.base_llm_http_handler" + ) as mock_handler, + ): + mock_handler.completion.return_value = "response" + + partner.completion( + model="meta/llama-3.1-405b-instruct-maas", + messages=[{"role": "user", "content": "hello"}], + model_response=MagicMock(), + print_verbose=lambda *a, **kw: None, + encoding=MagicMock(), + logging_obj=MagicMock(), + api_base=None, + optional_params={}, + custom_prompt_dict={}, + headers=None, + timeout=30.0, + litellm_params={}, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials='{"type": "service_account"}', + ) + + # _ensure_access_token should have been called on self + mock_ensure.assert_called_once_with( + credentials='{"type": "service_account"}', + project_id="test-project", + custom_llm_provider="vertex_ai", + ) + + def test_credential_cache_shared_across_calls(self): + """Two successive completion() calls should hit load_auth only once, + proving the credential cache on the VertexAIPartnerModels instance works.""" + partner = VertexAIPartnerModels() + + mock_creds = MagicMock() + mock_creds.token = "my-token" + mock_creds.expired = False + mock_creds.project_id = "proj" + mock_creds.quota_project_id = "proj" + + mock_vertexai = MagicMock() + mock_vertexai.preview = MagicMock() + mock_vertexai.preview.language_models = MagicMock() + + with ( + patch.dict(sys.modules, {"vertexai": mock_vertexai}), + patch.object( + partner, "load_auth", return_value=(mock_creds, "proj") + ) as mock_load, + patch( + "litellm.llms.vertex_ai.vertex_ai_partner_models.main.base_llm_http_handler" + ) as mock_handler, + ): + mock_handler.completion.return_value = "resp" + + common_kwargs = dict( + model="meta/llama-3.1-405b-instruct-maas", + messages=[{"role": "user", "content": "hi"}], + model_response=MagicMock(), + print_verbose=lambda *a, **kw: None, + encoding=MagicMock(), + logging_obj=MagicMock(), + api_base=None, + optional_params={}, + custom_prompt_dict={}, + headers=None, + timeout=30.0, + litellm_params={}, + vertex_project="proj", + vertex_location="us-central1", + vertex_credentials='{"type": "service_account"}', + ) + + partner.completion(**common_kwargs) + partner.completion(**common_kwargs) + + # load_auth should only be called once — second call uses cache + assert mock_load.call_count == 1