diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index f622098920c..b9f2359e9ea 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -139,6 +139,18 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None +def inferred_provider(model: str | None) -> str | None: + if not model: + return None + declared: Final = declared_authenticating_provider(model) + if declared is not None: + return declared + try: + return get_llm_provider(model=model)[1] + except Exception: # noqa: BLE001 # get_llm_provider raises for an unknown name, which then has no provider + return None + + def get_llm_provider( model: str, custom_llm_provider: str | None = None, diff --git a/litellm/router.py b/litellm/router.py index 56d27fedf28..9bf8c410bcb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7887,48 +7887,15 @@ class Router: """ return run_async_function(self.async_function_with_fallbacks, *args, **kwargs) - def _get_fallback_model_group_from_fallbacks( - self, - fallbacks: list[dict[str, list[str]]], - model_group: str | None = None, - ) -> list[str] | None: - """ - Returns the list of fallback models to use for a given model group - - If no fallback model group is found, returns None - - Example: - fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}] - model_group = "gpt-3.5-turbo" - returns: ["gpt-4"] - """ - if model_group is None: - return None - - fallback_model_group: list[str] | None = None - for item in fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}] - if list(item.keys())[0] == model_group: - fallback_model_group = item[model_group] - break - return fallback_model_group - def _get_fallback_model_group_for_lookup_groups( self, - fallbacks: list[dict[str, list[str]]], # mutable-ok: mirrors the sibling resolver's contract + fallbacks: list[dict[str, list[str]]], # mutable-ok: mirrors the shared resolver's contract lookup_groups: tuple[str, ...], - ) -> list[str] | None: # mutable-ok: mirrors the sibling resolver's contract - """First lookup group whose exact-key chain resolves (tier first, then requested group).""" - return next( - ( - resolved - for resolved in ( - self._get_fallback_model_group_from_fallbacks(fallbacks=fallbacks, model_group=group) - for group in lookup_groups - ) - if resolved is not None - ), - None, + ) -> list[str] | None: # mutable-ok: mirrors the shared resolver's contract + fallback_model_group, _ = get_fallback_model_group_for_lookup_groups( + fallbacks=fallbacks, lookup_groups=lookup_groups ) + return fallback_model_group def _get_first_default_fallback(self) -> str | None: """ diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 60585b3cc38..e0df7d1badf 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -11,6 +11,7 @@ import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.router_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, @@ -236,6 +237,13 @@ def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool: return False +def _provider_prefixed_model_group(model_group: str, fallback_keys: Sequence[str]) -> str | None: + if "/" in model_group or not any(key.endswith(f"/{model_group}") for key in fallback_keys): + return None + provider: Final = inferred_provider(model_group) + return f"{provider}/{model_group}" if provider else None + + PRE_ROUTING_SELECTED_MODEL_KEY: Final = "pre_routing_selected_model" _ROUTER_METADATA_BUCKETS: Final = ("metadata", "litellm_metadata") @@ -439,22 +447,26 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li Checks: - exact match - stripped model group match + - provider-prefixed model group match - generic fallback """ generic_fallback_idx: int | None = None stripped_model_fallback: list[str] | None = None fallback_model_group: list[str] | None = None + fallback_keys: Final = tuple(next(iter(item)) for item in fallbacks if isinstance(item, dict) and item) + prefixed_model_group: Final = _provider_prefixed_model_group(model_group, fallback_keys) ## check for specific model group-specific fallbacks for idx, item in enumerate(fallbacks): if isinstance(item, dict): - if list(item.keys())[0] == model_group: # check exact match + fallback_key = next(iter(item)) + if fallback_key == model_group: # check exact match fallback_model_group = item[model_group] break - elif _check_stripped_model_group( - model_group=model_group, fallback_key=list(item.keys())[0] + elif fallback_key == prefixed_model_group or _check_stripped_model_group( + model_group=model_group, fallback_key=fallback_key ): # check generic fallback - stripped_model_fallback = item[list(item.keys())[0]] - elif list(item.keys())[0] == "*": # check generic fallback + stripped_model_fallback = item[fallback_key] + elif fallback_key == "*": # check generic fallback generic_fallback_idx = idx elif isinstance(item, str): fallback_model_group = [item] diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index d5234e27ec6..cbb19cec9c2 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -8,7 +8,7 @@ from re import Match from typing import Final from litellm._logging import verbose_router_logger -from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, get_llm_provider +from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider class PatternUtils: @@ -218,18 +218,9 @@ class PatternMatchRouter: Returns: bool: True if pattern exists, False otherwise """ - provider: Final = ( - custom_llm_provider or declared_authenticating_provider(model) or self._resolved_provider(model) - ) + provider: Final = custom_llm_provider or inferred_provider(model) return self.route(model) or self.route(f"{provider}/{model}") - @staticmethod - def _resolved_provider(model: str | None) -> str | None: - try: - return get_llm_provider(model=model)[1] if model else None - except Exception: # noqa: BLE001 # get_llm_provider raises when the provider is unknown; the name then routes as-is - return None - def get_deployments_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> list[dict]: """ Get the deployments by pattern diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index a11f015743b..7332a533872 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -81,7 +81,6 @@ ignored_function_names = [ "_merge_tools_from_deployment", # Tested indirectly via _update_kwargs_with_deployment (test files lack "router" in name) "_invalidate_access_groups_cache", # Tested indirectly via set_model_list, upsert_model etc. (test files lack "router" in name) "has_buffered_provider_output", # Property, so its reads in test_router.py are never an ast.Call - "_resolved_provider", # Tested via get_pattern in test_pattern_match_deployments.py (file lacks "router" in name) "_request_header", # Tested through Claude Code session routing in test_router.py "_claude_code_session_router_cache_key", # Tested through Claude Code session routing in test_router.py "_delete_claude_code_session_router_binding", # Tested through Redis cleanup failure in test_router.py diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index bb389693311..7e593767ea9 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -449,16 +449,6 @@ def test_handle_mock_testing_rate_limit_error(model_list): ) -def test_get_fallback_model_group_from_fallbacks(model_list): - """Test if the '_get_fallback_model_group_from_fallbacks' function is working correctly""" - router = Router(model_list=model_list) - fallback_model_group_name = router._get_fallback_model_group_from_fallbacks( - model_group="gpt-5.5", - fallbacks=[{"gpt-5.5": "gpt-5-mini"}], - ) - assert fallback_model_group_name == "gpt-5-mini" - - @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_deployment_callback_on_success(sync_mode): diff --git a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py index ed8438bb1de..c067bc01e42 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/test_litellm/litellm_core_utils/test_get_llm_provider_logic.py @@ -5,8 +5,10 @@ import pytest import litellm from litellm import CustomLLM +from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.get_llm_provider_logic import ( get_llm_provider, + inferred_provider, is_registered_custom_provider, ) from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -78,3 +80,24 @@ def test_image_generation_fal_ai_egresses_to_global_api_base(monkeypatch: pytest client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))) litellm.image_generation(model="fal_ai/fal-ai/flux/schnell", prompt="a red kite", client=client) assert str(seen[0]).startswith("http://gateway.local/fal") + + +def test_inferred_provider_adopts_a_declared_authenticating_provider_without_resolving( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _oauth_tripwire(model: str, *args: object, **kwargs: object) -> None: + raise AssertionError(f"get_llm_provider would run the OAuth device flow for {model}") + + monkeypatch.setattr(get_llm_provider_logic, "get_llm_provider", _oauth_tripwire) + + assert inferred_provider("github_copilot/gpt-5.5") == "github_copilot" + + +def test_inferred_provider_matches_the_resolver_for_a_bare_model_name() -> None: + assert inferred_provider("claude-sonnet-4-6") == get_llm_provider(model="claude-sonnet-4-6")[1] + assert inferred_provider("gpt-5.5-pro") == get_llm_provider(model="gpt-5.5-pro")[1] + + +@pytest.mark.parametrize("model", ["some-unknown-model-xyz", "", None], ids=["unknown", "empty", "missing"]) +def test_inferred_provider_is_none_when_nothing_resolves(model: str | None) -> None: + assert inferred_provider(model) is None diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py index b363d3823ad..6cd613197ee 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_utils.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_utils.py @@ -15,8 +15,8 @@ import json import pytest import litellm +from litellm.litellm_core_utils import get_llm_provider_logic from litellm.proxy import proxy_server -from litellm.router_utils import pattern_match_deployments from .conftest import normalize # type: ignore[import-not-found] @@ -204,7 +204,7 @@ def test_supported_openai_params_never_runs_oauth_for_authenticating_providers(c raise AssertionError("get_llm_provider would run the OAuth device flow") monkeypatch.setattr(litellm, "get_llm_provider", _oauth_tripwire) - monkeypatch.setattr(pattern_match_deployments, "get_llm_provider", _oauth_tripwire) + monkeypatch.setattr(get_llm_provider_logic, "get_llm_provider", _oauth_tripwire) expected = litellm.get_supported_openai_params(model="gpt-4o", custom_llm_provider="github_copilot") with auth_as(): diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index b6ab21dfcef..972c04b209f 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -7,6 +7,7 @@ import httpx import pytest import litellm +from litellm.litellm_core_utils import get_llm_provider_logic from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure from litellm.router_utils.fallback_event_handlers import ( AttemptedFallbackTargets, @@ -1340,3 +1341,61 @@ class TestHasUnattemptedFallbackTarget: assert has_unattempted_fallback_target(["fb1", "fb3"], {"attempted_targets": attempted}) is True assert has_unattempted_fallback_target(["fb1"], {}) is True assert has_unattempted_fallback_target(None, {}) is False + + +def test_get_fallback_model_group_matches_provider_prefixed_key(): + """A bare model group routed via a wildcard (e.g. "gpt-4o" through + "openai/*") must match a fallback keyed on the provider-prefixed name, + which is the form the Admin UI offers for wildcard routes.""" + fallbacks = [{"openai/gpt-4o": ["claude-3-haiku"]}] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group="gpt-4o") + + assert fallback_model_group == ["claude-3-haiku"] + + +def test_get_fallback_model_group_exact_match_beats_prefixed_match(): + fallbacks = [ + {"openai/gpt-4o": ["claude-3-haiku"]}, + {"gpt-4o": ["gemini-1.5-flash"]}, + ] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group="gpt-4o") + + assert fallback_model_group == ["gemini-1.5-flash"] + + +def test_get_fallback_model_group_prefixed_match_ignores_unknown_models(): + """Provider inference fails for unknown bare names - the lookup must not + raise and must fall through to the generic fallback.""" + fallbacks = [ + {"openai/some-model": ["claude-3-haiku"]}, + {"*": ["gemini-1.5-flash"]}, + ] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group="some-unknown-model-xyz") + + assert fallback_model_group == ["gemini-1.5-flash"] + + +def test_get_fallback_model_group_prefixed_match_skips_prefixed_model_group(): + """An already-prefixed model group must not double-prefix.""" + fallbacks = [{"openai/openai/gpt-4o": ["claude-3-haiku"]}] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4o") + + assert fallback_model_group is None + + +def test_get_fallback_model_group_never_resolves_a_provider_without_a_prefixed_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An alias-style group name has no provider, and resolving it prints the SDK's provider-list banner, + so the lookup only infers a provider when some key is spelled /.""" + + resolver: Final = MagicMock(return_value=("my-alias", "openai", None, None)) + monkeypatch.setattr(get_llm_provider_logic, "get_llm_provider", resolver) + fallbacks: Final = [{"gpt-5.5-pro": ["claude-sonnet-4-6"]}, {"*": ["gpt-5.5-mini"]}] + + assert get_fallback_model_group(fallbacks=fallbacks, model_group="my-alias") == (["gpt-5.5-mini"], 1) + resolver.assert_not_called() diff --git a/tests/test_litellm/router_utils/test_pattern_match_deployments.py b/tests/test_litellm/router_utils/test_pattern_match_deployments.py index f9d9345cd26..ea09bcc2e15 100644 --- a/tests/test_litellm/router_utils/test_pattern_match_deployments.py +++ b/tests/test_litellm/router_utils/test_pattern_match_deployments.py @@ -4,7 +4,7 @@ from __future__ import annotations from unittest.mock import Mock -from litellm.router_utils import pattern_match_deployments +from litellm.litellm_core_utils import get_llm_provider_logic from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils @@ -26,7 +26,7 @@ def test_get_pattern_never_resolves_declared_authenticating_providers(monkeypatc resolution_attempts.append(model) raise AssertionError("get_llm_provider would run the OAuth device flow") - monkeypatch.setattr(pattern_match_deployments, "get_llm_provider", _oauth_tripwire) + monkeypatch.setattr(get_llm_provider_logic, "get_llm_provider", _oauth_tripwire) unmatched_router = PatternMatchRouter() unmatched_router.add_pattern("anthropic/*", _wildcard_deployment("anthropic/*")) @@ -49,7 +49,7 @@ def test_get_pattern_bare_provider_name_never_matches_that_providers_wildcard(mo def _unknown_provider(model, *args, **kwargs): raise ValueError(f"unknown provider for {model}") - monkeypatch.setattr(pattern_match_deployments, "get_llm_provider", _unknown_provider) + monkeypatch.setattr(get_llm_provider_logic, "get_llm_provider", _unknown_provider) router = PatternMatchRouter() router.add_pattern("github_copilot/*", _wildcard_deployment("github_copilot/*")) assert router.get_pattern("github_copilot") is None @@ -63,7 +63,7 @@ def test_get_pattern_missing_model_returns_none(monkeypatch): def _unknown_provider(model, *args, **kwargs): raise ValueError(f"unknown provider for {model}") - monkeypatch.setattr(pattern_match_deployments, "get_llm_provider", _unknown_provider) + monkeypatch.setattr(get_llm_provider_logic, "get_llm_provider", _unknown_provider) router = PatternMatchRouter() router.add_pattern("openai/*", _wildcard_deployment("openai/*")) assert router.get_pattern(None) is None @@ -71,7 +71,7 @@ def test_get_pattern_missing_model_returns_none(monkeypatch): def test_get_pattern_still_resolves_unqualified_names(monkeypatch): monkeypatch.setattr( - pattern_match_deployments, + get_llm_provider_logic, "get_llm_provider", lambda model, **kwargs: (model, "openai", None, None), ) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index f985b212b01..82122da15dc 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -17862,3 +17862,95 @@ def test_access_windows_filter_reserved_deployments_method(): request_team_id="team-a", ) ] == ["reserved-deployment", "open-deployment"] + + +@pytest.mark.asyncio +async def test_bare_model_group_served_by_wildcard_deployment_uses_provider_prefixed_fallback_key() -> None: + """Claude Code sends the bare "claude-sonnet-4-6" to /v1/messages; routing serves it through the + "anthropic/*" wildcard, so a fallback keyed the way that wildcard is written ("anthropic/claude-sonnet-4-6", + which is what the Admin UI offers) must catch the failure instead of surfacing the provider error.""" + router = litellm.Router( + model_list=[ + { + "model_name": "anthropic/*", + "litellm_params": { + "model": "anthropic/*", + "api_key": "sk-fake", + "mock_response": "litellm.InternalServerError", + }, + }, + { + "model_name": "openai/gpt-5.5-pro", + "litellm_params": { + "model": "openai/gpt-5.5-pro", + "api_key": "sk-fake", + "mock_response": "served by the fallback", + }, + }, + ], + fallbacks=[{"anthropic/claude-sonnet-4-6": ["openai/gpt-5.5-pro"]}], + num_retries=0, + ) + + result = await router.aanthropic_messages( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hi"}], + max_tokens=64, + ) + + assert result["content"][0]["text"] == "served by the fallback" + + +@pytest.mark.asyncio +async def test_bare_model_group_served_by_wildcard_deployment_uses_provider_prefixed_context_window_fallback_key() -> None: + """The context-window chain is keyed the same way the ordinary chain is, so a key spelled like the + wildcard deployment ("anthropic/claude-sonnet-4-6") must catch the bare group's context-window error too.""" + router = litellm.Router( + model_list=[ + { + "model_name": "anthropic/*", + "litellm_params": { + "model": "anthropic/*", + "api_key": "sk-fake", + "mock_response": "litellm.ContextWindowExceededError", + }, + }, + { + "model_name": "openai/gpt-5.5-pro", + "litellm_params": { + "model": "openai/gpt-5.5-pro", + "api_key": "sk-fake", + "mock_response": "served by the context window fallback", + }, + }, + ], + context_window_fallbacks=[{"anthropic/claude-sonnet-4-6": ["openai/gpt-5.5-pro"]}], + num_retries=0, + ) + + result = await router.aanthropic_messages( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "hi"}], + max_tokens=64, + ) + + assert result["content"][0]["text"] == "served by the context window fallback" + + +def test_bare_model_group_served_by_wildcard_deployment_has_provider_prefixed_content_policy_fallback() -> None: + router = litellm.Router( + model_list=[ + { + "model_name": "anthropic/*", + "litellm_params": {"model": "anthropic/*", "api_key": "sk-fake"}, + }, + { + "model_name": "openai/gpt-5.5-pro", + "litellm_params": {"model": "openai/gpt-5.5-pro", "api_key": "sk-fake"}, + }, + ], + content_policy_fallbacks=[{"anthropic/claude-sonnet-4-6": ["openai/gpt-5.5-pro"]}], + ) + + assert router._has_content_policy_fallback("claude-sonnet-4-6", {}) is True + assert router._has_content_policy_fallback("claude-haiku-4-5", {}) is False