diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 19c1d5c414c..6ace8c42505 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -158,6 +158,22 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None +def declared_provider(model: str, custom_llm_provider: str | None = None) -> str | None: + """The provider this pair names outright, or None when only get_llm_provider can tell. + + get_llm_provider reads a named provider's API key and base on the way to returning it, and a hosted + secret manager answers each read with a network call, so a caller that needs only the name reads it here. + """ + if custom_llm_provider: + return custom_llm_provider + prefix, slash, _ = model.partition("/") + if not slash: + return None + if prefix in litellm.provider_list or JSONProviderRegistry.exists(prefix) or is_registered_custom_provider(prefix): + return prefix + return None + + def resolve_model_and_provider_for_metadata(model: str, custom_llm_provider: str | None = None) -> tuple[str, str]: declared: Final = declared_authenticating_provider(model, custom_llm_provider) if declared is not None: diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 3731abe3134..68c11688c9c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -50,7 +50,7 @@ from litellm.constants import ( from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix -from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata +from litellm.litellm_core_utils.get_llm_provider_logic import declared_provider, inferred_provider from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, @@ -3927,11 +3927,9 @@ def _vertex_upstream_model(deployment: DeploymentTypedDict) -> str | None: if "model" not in litellm_params: return None upstream: Final = litellm_params["model"] - try: - _, provider = resolve_model_and_provider_for_metadata( - model=upstream, custom_llm_provider=litellm_params.get("custom_llm_provider") - ) - except litellm.exceptions.BadRequestError: + declared: Final = declared_provider(upstream, litellm_params.get("custom_llm_provider")) + provider: Final = declared or inferred_provider(upstream) + if provider is None: return upstream return upstream.removeprefix(f"{provider}/") if provider in VERTEX_LIVE_PROVIDERS else None diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index e388c658200..659be2666bc 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -34,6 +34,8 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, WEBSOCKET_CLOSE_REASON_MAX_BYTES from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.custom_secret_manager import CustomSecretManager +from litellm.llms.custom_llm import CustomLLM from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth @@ -65,6 +67,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.types import utils as types_utils from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -8083,6 +8086,61 @@ async def test_vertex_live_setup_frame_never_refreshes_the_token_of_a_chatgpt_gr assert _policy_close_reason(websocket) is None +class SecretReadRecorder(CustomSecretManager): + """A hosted secret manager that answers every read and records the name it was asked for""" + + def __init__(self) -> None: + self.names_read = MagicMock() + + def sync_read_secret( + self, secret_name: str, optional_params: dict | None = None, timeout: float | httpx.Timeout | None = None + ) -> str: + self.names_read(secret_name) + return "a-value" + + async def async_read_secret( + self, secret_name: str, optional_params: dict | None = None, timeout: float | httpx.Timeout | None = None + ) -> str: + return self.sync_read_secret(secret_name) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "other_group", + [ + {"model": "groq/llama-4-scout"}, + {"model": "llama-4-scout", "custom_llm_provider": "groq"}, + {"model": "veniceai/venice-uncensored"}, + {"model": "in-house/assistant"}, + ], + ids=["provider-prefix", "custom-llm-provider", "json-provider", "registered-custom-provider"], +) +async def test_vertex_live_setup_frame_never_reads_the_secret_manager_for_another_providers_group( + monkeypatch: pytest.MonkeyPatch, other_group: dict[str, str] +): + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "in-house", "custom_handler": CustomLLM()}]) + llm_router: Final = litellm.Router( + model_list=[ + {"model_name": "other", "litellm_params": other_group}, + {"model_name": "native-audio", "litellm_params": {"model": f"vertex_ai/{VERTEX_LIVE_GROUP}"}}, + ] + ) + secret_manager: Final = SecretReadRecorder() + monkeypatch.setattr(litellm, "secret_manager_client", secret_manager) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + + websocket, upstream_ws = await _run_vertex_live_gated_passthrough( + [_setup_frame("native-audio")], + UserAPIKeyAuth(token="hashed", models=["native-audio"]), + llm_router=llm_router, + ) + + secret_manager.names_read.assert_not_called() + upstream_ws.send.assert_awaited_once() + assert _policy_close_reason(websocket) is None + + @pytest.mark.asyncio @pytest.mark.parametrize( "valid_token",