mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge 3245454f36 into 252c71c0b2
This commit is contained in:
commit
d77cae0455
3 changed files with 216 additions and 1 deletions
|
|
@ -1517,6 +1517,42 @@ def _timing_values(
|
|||
return getattr(logging_obj, "response_timing_metrics", None) or {} # mutable-ok: empty fallback
|
||||
|
||||
|
||||
async def _filter_fallback_models_by_key_access(
|
||||
fallback_models: Sequence[object],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
llm_router: Router,
|
||||
) -> Sequence[object]:
|
||||
"""
|
||||
Drop fallback targets the key is not allowed to call.
|
||||
|
||||
The rate-limit pre-call fallback swaps the request model after auth
|
||||
already ran for the original model. Without this filter, a key restricted
|
||||
to one model group would be rerouted to any model group the config (or
|
||||
the key's router_settings) lists as a fallback.
|
||||
"""
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
|
||||
|
||||
allowed: list[object] = [] # mutable-ok: builds the filtered fallback list this function returns
|
||||
for fallback_model in fallback_models:
|
||||
if not isinstance(fallback_model, str):
|
||||
# dict-form fallback entries ({model, messages}) re-run auth below
|
||||
# via common_processing_pre_call_logic's model extraction; keep them.
|
||||
allowed.append(fallback_model)
|
||||
continue
|
||||
try:
|
||||
await can_key_call_resolved_model(
|
||||
model=fallback_model,
|
||||
llm_model_list=None,
|
||||
valid_token=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
except ProxyException:
|
||||
continue
|
||||
allowed.append(fallback_model)
|
||||
return allowed
|
||||
|
||||
|
||||
class ProxyBaseLLMRequestProcessing:
|
||||
def __init__(self, data: dict):
|
||||
self.data = data
|
||||
|
|
@ -2077,8 +2113,20 @@ class ProxyBaseLLMRequestProcessing:
|
|||
fallback_models,
|
||||
)
|
||||
|
||||
# Re-run the key/team model allowlist for each fallback target.
|
||||
# Auth already ran for the original model; swapping the model here
|
||||
# must not widen access to model groups the key is denied. Mirrors
|
||||
# the client-side fallback checks in _enforce_key_and_fallback_model_access.
|
||||
allowed_fallback_models: Final = await _filter_fallback_models_by_key_access(
|
||||
fallback_models=fallback_models,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
if not allowed_fallback_models:
|
||||
raise
|
||||
|
||||
try:
|
||||
for fallback_model in fallback_models:
|
||||
for fallback_model in allowed_fallback_models:
|
||||
if fallback_model == original_model:
|
||||
continue
|
||||
self.data["model"] = fallback_model
|
||||
|
|
|
|||
|
|
@ -446,6 +446,55 @@ def creates_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
|
|||
return getattr(kwargs.get("original_function"), "__name__", None) in PROVIDER_SCOPED_CREATION_FUNCTION_NAMES
|
||||
|
||||
|
||||
async def _proxy_key_allows_fallback_model(
|
||||
litellm_router: LitellmRouter,
|
||||
kwargs: Mapping[str, object],
|
||||
fallback_entry: str | Mapping[str, object],
|
||||
) -> bool:
|
||||
"""
|
||||
Re-run the proxy's model authorization for a server-side fallback target.
|
||||
|
||||
A router-configured fallback swaps the model after auth already ran;
|
||||
without this check a key restricted to one model group would be served
|
||||
by any shared deployment the config lists as a fallback for it.
|
||||
Non-proxy (SDK) callers carry no auth object and are unaffected.
|
||||
"""
|
||||
metadata: Final = kwargs.get("metadata")
|
||||
litellm_metadata: Final = kwargs.get("litellm_metadata")
|
||||
auth: Final = (
|
||||
metadata.get("user_api_key_auth") if isinstance(metadata, Mapping) else None
|
||||
) or (
|
||||
litellm_metadata.get("user_api_key_auth")
|
||||
if isinstance(litellm_metadata, Mapping)
|
||||
else None
|
||||
)
|
||||
if auth is None:
|
||||
return True
|
||||
model: Final = _get_fallback_target_model_group(fallback_entry)
|
||||
if model is None:
|
||||
return True
|
||||
|
||||
# Inline import: the proxy layer is only reachable when the proxy populated
|
||||
# the metadata above, and a module-level import would be circular.
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
|
||||
|
||||
try:
|
||||
await can_key_call_resolved_model(
|
||||
model=model,
|
||||
llm_model_list=None,
|
||||
valid_token=auth,
|
||||
llm_router=litellm_router,
|
||||
)
|
||||
except ProxyException:
|
||||
verbose_router_logger.info(
|
||||
"Skipping fallback to model_group = %s: not allowed for this key.",
|
||||
mask_sensitive_structure(model),
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
async def run_async_fallback(
|
||||
*args: object,
|
||||
litellm_router: LitellmRouter,
|
||||
|
|
@ -529,6 +578,12 @@ async def run_async_fallback(
|
|||
continue
|
||||
attempted.record(attempt_key)
|
||||
try:
|
||||
if not await _proxy_key_allows_fallback_model(
|
||||
litellm_router=litellm_router,
|
||||
kwargs=kwargs,
|
||||
fallback_entry=mg,
|
||||
):
|
||||
continue
|
||||
# LOGGING
|
||||
kwargs = litellm_router.log_retry(kwargs=kwargs, e=original_exception)
|
||||
verbose_router_logger.info("Falling back to model_group = %s", mask_sensitive_structure(mg))
|
||||
|
|
|
|||
112
tests/router_unit_tests/test_fallback_model_scope_check.py
Normal file
112
tests/router_unit_tests/test_fallback_model_scope_check.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
"""Regression: server-side fallbacks re-check model authorization.
|
||||
|
||||
A key restricted to model-a should not be served by model-b via a
|
||||
router-configured fallback, regardless of which metadata bucket carries
|
||||
the auth object (metadata vs litellm_metadata).
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.router_utils.fallback_event_handlers import (
|
||||
_proxy_key_allows_fallback_model,
|
||||
)
|
||||
|
||||
|
||||
def _make_kwargs(metadata=None, litellm_metadata=None):
|
||||
kwargs = {}
|
||||
if metadata is not None:
|
||||
kwargs["metadata"] = metadata
|
||||
if litellm_metadata is not None:
|
||||
kwargs["litellm_metadata"] = litellm_metadata
|
||||
return kwargs
|
||||
|
||||
|
||||
def _make_auth(team_models=None, key_models=None):
|
||||
auth = MagicMock()
|
||||
auth.team_models = team_models or []
|
||||
auth.team_model_aliases = None
|
||||
auth.team_id = None
|
||||
auth.models = key_models or []
|
||||
auth.config = None
|
||||
return auth
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_auth_returns_true():
|
||||
"""SDK callers (no auth object) are unaffected."""
|
||||
kwargs = _make_kwargs(metadata={})
|
||||
assert await _proxy_key_allows_fallback_model(
|
||||
litellm_router=MagicMock(), kwargs=kwargs, fallback_entry="model-b"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_bucket_auth(monkeypatch):
|
||||
"""Auth resolved from kwargs['metadata'] (the standard proxy path)."""
|
||||
from litellm.proxy.auth import auth_checks
|
||||
|
||||
check = AsyncMock(side_effect=auth_checks.ProxyException(message="denied"))
|
||||
monkeypatch.setattr(auth_checks, "can_key_call_resolved_model", check)
|
||||
|
||||
kwargs = _make_kwargs(metadata={"user_api_key_auth": _make_auth()})
|
||||
result = await _proxy_key_allows_fallback_model(
|
||||
litellm_router=MagicMock(), kwargs=kwargs, fallback_entry="model-b"
|
||||
)
|
||||
assert result is False
|
||||
check.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_metadata_bucket_auth(monkeypatch):
|
||||
"""Auth resolved from kwargs['litellm_metadata'] (the /v1/messages and
|
||||
/v1/responses path). Without this, those routes skip the check entirely."""
|
||||
from litellm.proxy.auth import auth_checks
|
||||
|
||||
check = AsyncMock(side_effect=auth_checks.ProxyException(message="denied"))
|
||||
monkeypatch.setattr(auth_checks, "can_key_call_resolved_model", check)
|
||||
|
||||
kwargs = _make_kwargs(litellm_metadata={"user_api_key_auth": _make_auth()})
|
||||
result = await _proxy_key_allows_fallback_model(
|
||||
litellm_router=MagicMock(), kwargs=kwargs, fallback_entry="model-b"
|
||||
)
|
||||
assert result is False
|
||||
check.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metadata_takes_precedence_over_litellm_metadata(monkeypatch):
|
||||
"""When both buckets carry auth, the metadata bucket wins (standard path)."""
|
||||
from litellm.proxy.auth import auth_checks
|
||||
|
||||
check = AsyncMock() # no exception = allowed
|
||||
monkeypatch.setattr(auth_checks, "can_key_call_resolved_model", check)
|
||||
|
||||
kwargs = _make_kwargs(
|
||||
metadata={"user_api_key_auth": _make_auth()},
|
||||
litellm_metadata={"user_api_key_auth": _make_auth()},
|
||||
)
|
||||
result = await _proxy_key_allows_fallback_model(
|
||||
litellm_router=MagicMock(), kwargs=kwargs, fallback_entry="model-b"
|
||||
)
|
||||
assert result is True
|
||||
check.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_allowed_model_returns_true(monkeypatch):
|
||||
"""When the resolved-model check passes, the fallback proceeds."""
|
||||
from litellm.proxy.auth import auth_checks
|
||||
|
||||
check = AsyncMock() # no exception = allowed
|
||||
monkeypatch.setattr(auth_checks, "can_key_call_resolved_model", check)
|
||||
|
||||
kwargs = _make_kwargs(metadata={"user_api_key_auth": _make_auth()})
|
||||
result = await _proxy_key_allows_fallback_model(
|
||||
litellm_router=MagicMock(), kwargs=kwargs, fallback_entry="model-b"
|
||||
)
|
||||
assert result is True
|
||||
check.assert_called_once_with(
|
||||
model="model-b", llm_model_list=None, valid_token=kwargs["metadata"]["user_api_key_auth"], llm_router=pytest.ANY
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue