diff --git a/litellm/router.py b/litellm/router.py index 98a5ab2a5fd..9cece2014af 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -112,6 +112,7 @@ from litellm.router_utils.common_utils import ( filter_web_search_deployments, resolve_model_group_alias, truncate_fallback_error_detail, + warn_on_provider_credential_mismatch, ) from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.cooldown_handlers import ( @@ -7540,6 +7541,7 @@ class Router: """ try: litellm_params: Final[LiteLLM_Params] = LiteLLM_Params(**_litellm_params) + warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=_litellm_params) deployment = Deployment( **deployment_info, model_name=_model_name, @@ -8232,6 +8234,11 @@ class Router: if _deployment_model_id and self.has_model_id(_deployment_model_id): return None + warn_on_provider_credential_mismatch( + model_name=deployment.model_name, + litellm_params=deployment.litellm_params.model_dump(exclude_none=True), + ) + # add to model list _deployment: Final = deployment.to_json(exclude_none=True) # initialize client diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 6fad2dd31e9..280a7defcf8 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -1,15 +1,18 @@ import hashlib import json from collections.abc import Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIFileObject -from litellm._logging import verbose_logger +from litellm._logging import verbose_logger, verbose_router_logger from litellm.constants import ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.types.router import CredentialLiteLLMParams +from litellm.types.utils import LlmProviders def _is_proxy_admin_request(request_kwargs: Mapping[str, object] | None) -> bool: @@ -210,3 +213,77 @@ def filter_web_search_deployments( if len(healthy_deployments) > 0 and len(final_deployments) == 0: verbose_logger.warning("No deployments support web search for request") return final_deployments + + +# Credential params that only one provider family reads, paired with the providers +# that read them. A deployment carrying them while resolving elsewhere is almost +# always a missing route prefix: `model: claude-sonnet-5` with `aws_region_name` +# set resolves to the first-party Anthropic API, silently ignores the AWS +# credentials, and 401s at request time. +_AWS_PROVIDERS: Final = frozenset( + provider.value for provider in LlmProviders if provider.value.startswith(("bedrock", "sagemaker")) +) +_VERTEX_PROVIDERS: Final = frozenset( + provider.value for provider in LlmProviders if provider.value.startswith("vertex_ai") +) + +PROVIDER_SCOPED_CREDENTIAL_PARAMS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "aws_access_key_id": _AWS_PROVIDERS, + "aws_profile_name": _AWS_PROVIDERS, + "aws_region_name": _AWS_PROVIDERS, + "aws_role_name": _AWS_PROVIDERS, + "aws_secret_access_key": _AWS_PROVIDERS, + "aws_session_name": _AWS_PROVIDERS, + "aws_session_token": _AWS_PROVIDERS, + "aws_web_identity_token": _AWS_PROVIDERS, + "vertex_credentials": _VERTEX_PROVIDERS, + "vertex_location": _VERTEX_PROVIDERS, + "vertex_project": _VERTEX_PROVIDERS, + } +) + + +def warn_on_provider_credential_mismatch(model_name: str, litellm_params: Mapping[str, object]) -> str | None: + """ + Warn when a deployment carries one provider's credentials but resolves to another. + + Returns the warning text (for tests), or None when the deployment is consistent + or its provider cannot be resolved. Never raises: a deployment litellm cannot + classify is left alone rather than blocking router startup. + + Only inline credential params are examined. A deployment that sources them + through ``litellm_credential_name`` resolves them after registration, so it + carries none of these keys here and is left alone rather than warned about + on incomplete information. + """ + model: Final = litellm_params.get("model") + if not isinstance(model, str) or not model: + return None + scoped: Final = tuple(param for param in PROVIDER_SCOPED_CREDENTIAL_PARAMS if litellm_params.get(param) is not None) + if not scoped: + return None + custom_llm_provider: Final = litellm_params.get("custom_llm_provider") + try: + _, resolved_provider, _, _ = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + ) + except BadRequestError: + return None + mismatched: Final = sorted( + param for param in scoped if resolved_provider not in PROVIDER_SCOPED_CREDENTIAL_PARAMS[param] + ) + if not mismatched: + return None + expected: Final = sorted( + {provider for param in mismatched for provider in PROVIDER_SCOPED_CREDENTIAL_PARAMS[param]} + ) + warning: Final = ( + f"Deployment '{model_name}' sets {mismatched} but 'model={model}' resolves to provider " + f"'{resolved_provider}', which ignores them. Those params are read by {expected}, so this is " + f"usually a missing route prefix (e.g. '{expected[0]}/{model}'); as written the request goes to " + f"'{resolved_provider}' and will fail on that provider's credentials." + ) + verbose_router_logger.warning(warning) + return warning diff --git a/tests/test_litellm/router_utils/test_router_utils_common_utils.py b/tests/test_litellm/router_utils/test_router_utils_common_utils.py index 0d063ad14f5..30f658d7ea2 100644 --- a/tests/test_litellm/router_utils/test_router_utils_common_utils.py +++ b/tests/test_litellm/router_utils/test_router_utils_common_utils.py @@ -1,3 +1,4 @@ +import logging from typing import Dict, List, Optional, Union from unittest.mock import Mock @@ -13,6 +14,8 @@ from litellm.router_utils.common_utils import ( filter_web_search_deployments, resolve_model_group_alias, truncate_fallback_error_detail, + PROVIDER_SCOPED_CREDENTIAL_PARAMS, + warn_on_provider_credential_mismatch, ) @@ -584,3 +587,172 @@ class TestTruncateFallbackErrorDetail: to stay small enough that a walk over many model groups cannot compound it into an output volume that starves the process.""" assert len(truncate_fallback_error_detail("x" * 1_000_000)) < 3_000 + + +class TestWarnOnProviderCredentialMismatch: + """A deployment that carries one provider's credentials while resolving to + another is silently broken: litellm ignores the credentials and sends the + request to the resolved provider, which 401s. The classic shape is a bedrock + model group where one entry lost its route prefix, which fails only on the + requests the router happens to send to that entry.""" + + def test_warns_when_aws_params_sit_on_an_anthropic_model(self): + warning = warn_on_provider_credential_mismatch( + model_name="claude-sonnet-5", + litellm_params={"model": "claude-sonnet-5", "aws_region_name": "eu-central-1"}, + ) + + assert warning is not None + assert "aws_region_name" in warning + assert "anthropic" in warning + assert "bedrock/claude-sonnet-5" in warning + + def test_silent_when_the_prefix_is_present(self): + assert ( + warn_on_provider_credential_mismatch( + model_name="claude-sonnet-5", + litellm_params={ + "model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "aws_region_name": "eu-central-1", + }, + ) + is None + ) + + def test_silent_when_custom_llm_provider_supplies_the_route(self): + """An operator may name the provider explicitly instead of prefixing the + model; that is consistent and must not warn.""" + assert ( + warn_on_provider_credential_mismatch( + model_name="claude-sonnet-5", + litellm_params={ + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "custom_llm_provider": "bedrock", + "aws_region_name": "eu-central-1", + }, + ) + is None + ) + + def test_silent_when_no_provider_scoped_credentials_are_set(self): + assert ( + warn_on_provider_credential_mismatch( + model_name="gpt-5.5", litellm_params={"model": "gpt-5.5"} + ) + is None + ) + + def test_vertex_params_name_vertex_not_bedrock(self): + """The hint must follow the params that were actually set, otherwise it + sends the operator to the wrong prefix.""" + warning = warn_on_provider_credential_mismatch( + model_name="claude-on-vertex", + litellm_params={"model": "claude-sonnet-5", "vertex_project": "my-project"}, + ) + + assert warning is not None + assert "vertex_ai/claude-sonnet-5" in warning + assert "bedrock" not in warning + + def test_silent_for_a_model_litellm_cannot_classify(self): + """An unresolvable model must not warn and must not raise: this runs on + the router startup path, so a wrong guess would spam every boot.""" + assert ( + warn_on_provider_credential_mismatch( + model_name="mystery", + litellm_params={"model": "not-a-real-provider-model-xyz", "aws_region_name": "us-east-1"}, + ) + is None + ) + + def test_router_warns_for_a_config_shaped_model_list(self, caplog): + """The whole point is that this fires where operators declare models, so + drive Router rather than the helper.""" + with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): + Router( + model_list=[ + { + "model_name": "claude-sonnet-5", + "litellm_params": { + "model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "aws_region_name": "us-east-1", + }, + }, + { + "model_name": "claude-sonnet-5", + "litellm_params": { + "model": "claude-sonnet-5", + "aws_region_name": "us-east-1", + }, + }, + ] + ) + + mismatch_warnings = [r for r in caplog.records if "resolves to provider" in r.getMessage()] + assert len(mismatch_warnings) == 1, ( + "exactly the prefix-less deployment should warn; " + f"got {[r.getMessage() for r in mismatch_warnings]}" + ) + assert "aws_region_name" in mismatch_warnings[0].getMessage() + + @pytest.mark.parametrize( + "model", + [ + "bedrock/mantle/anthropic.claude-sonnet-4-5-20250929-v1:0", + "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "sagemaker/my-endpoint", + ], + ) + def test_silent_for_every_aws_family_route(self, model): + """The AWS family is wider than 'bedrock': mantle, sagemaker and the + sagemaker variants all read aws_* legitimately. Warning on any of them + would tell an operator to 'fix' a working deployment, so the provider + set is derived from LlmProviders rather than hand-listed.""" + assert ( + warn_on_provider_credential_mismatch( + model_name="aws-deployment", + litellm_params={"model": model, "aws_region_name": "us-east-1"}, + ) + is None + ) + + def test_every_aws_family_provider_is_covered(self): + """Pins the derivation itself: a newly added bedrock_*/sagemaker_* provider + must join the set automatically, or it starts drawing false warnings.""" + from litellm.types.utils import LlmProviders + + aws_family = {p.value for p in LlmProviders if p.value.startswith(("bedrock", "sagemaker"))} + assert aws_family <= PROVIDER_SCOPED_CREDENTIAL_PARAMS["aws_region_name"] + assert {"bedrock", "bedrock_mantle", "sagemaker", "sagemaker_chat", "sagemaker_nova"} <= aws_family + + def test_silent_when_credentials_come_from_a_named_credential(self): + """Named credentials resolve after registration, so the params are absent + here. Warning on that absence would fire on every such deployment.""" + assert ( + warn_on_provider_credential_mismatch( + model_name="claude-sonnet-5", + litellm_params={ + "model": "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0", + "litellm_credential_name": "my-aws-creds", + }, + ) + is None + ) + + @pytest.mark.parametrize("provider", ["bedrock_mantle", "sagemaker_nova"]) + def test_silent_for_aws_providers_named_explicitly(self, provider): + """The false-positive shape: an operator names a less common AWS provider + directly, so the model string carries no route prefix to key off. A + hand-listed provider set misses these and tells them to 'fix' a working + deployment by prefixing it with bedrock/.""" + assert ( + warn_on_provider_credential_mismatch( + model_name="aws-deployment", + litellm_params={ + "model": "anthropic.claude-sonnet-4-5-20250929-v1:0", + "custom_llm_provider": provider, + "aws_region_name": "us-east-1", + }, + ) + is None + )