fix(router): keep deployment id grants out of the sdk proxy import path

Check caller grants through a runtime protocol instead of importing UserAPIKeyAuth
into router.py, and pass caller auth through the remaining public-name lookups
(model info, token-array embeddings, file size limits, provider resolution) so a
deployment id grant resolves the granted deployment rather than its first sibling

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
ryan 2026-09-19 01:25:47 +00:00
parent 796576d8df
commit 8ce2a7a149
10 changed files with 151 additions and 10 deletions

View file

@ -14,7 +14,7 @@ from litellm.constants import (
CLIENT_REQUESTED_MODEL_SCOPE_KEY,
MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB,
)
from litellm.proxy._types import ProxyException
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
get_metadata_variable_name_from_kwargs,
)
@ -329,6 +329,7 @@ def check_file_size_under_limit(
request_data: dict,
file: UploadFile,
router_model_names: Collection[str],
user_api_key_dict: UserAPIKeyAuth | None = None,
) -> bool:
"""
Check if any files passed in request are under max_file_size_mb
@ -353,7 +354,7 @@ def check_file_size_under_limit(
if llm_router is not None and request_data["model"] in router_model_names:
try:
deployment: Final[Deployment | None] = llm_router.get_deployment_by_model_group_name(
model_group_name=request_data["model"]
model_group_name=request_data["model"], user_api_key_auth=user_api_key_dict
)
if (
deployment

View file

@ -2692,6 +2692,7 @@ def _apply_credential_overrides_from_model_config(
llm_router=llm_router,
model_name=model_name,
pre_alias_model_name=pre_alias_model_name,
user_api_key_dict=user_api_key_dict,
)
credential_name: Final = _resolve_credential_from_model_config(
@ -2732,6 +2733,7 @@ def _resolve_provider_from_deployment(
llm_router: Router,
model_name: str,
pre_alias_model_name: str | None = None,
user_api_key_dict: UserAPIKeyAuth | None = None,
) -> str | None:
"""
Resolve a provider hint from the deployment's litellm_params when the
@ -2747,7 +2749,9 @@ def _resolve_provider_from_deployment(
for name in candidates:
try:
deployment = llm_router.get_deployment_by_model_group_name(model_group_name=name)
deployment = llm_router.get_deployment_by_model_group_name(
model_group_name=name, user_api_key_auth=user_api_key_dict
)
except Exception:
deployment = None
if deployment is None:

View file

@ -11051,7 +11051,9 @@ async def model_info(
if llm_router is None:
raise HTTPException(status_code=500, detail="Router not initialized")
deployment: Final = llm_router.get_deployment_by_model_group_name(resolved_model_id)
deployment: Final = llm_router.get_deployment_by_model_group_name(
resolved_model_id, user_api_key_auth=user_api_key_dict
)
if deployment is None:
raise HTTPException(
status_code=404,
@ -11495,7 +11497,9 @@ async def embeddings(
# check if provider accept list of tokens as input - e.g. for langchain integration
if llm_router is not None and data.get("model") in router_model_names:
# Use router's O(1) lookup instead of O(N) iteration through llm_model_list
deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=data["model"])
deployment: Final = llm_router.get_deployment_by_model_group_name(
model_group_name=data["model"], user_api_key_auth=user_api_key_dict
)
if deployment is not None:
litellm_params: Final = deployment.get("litellm_params", {}) or {}
litellm_model: Final = litellm_params.get("model", "")
@ -11882,6 +11886,7 @@ async def audio_transcriptions(
request_data=data,
file=file,
router_model_names=router_model_names,
user_api_key_dict=user_api_key_dict,
)
file_content: Final = await file.read()

View file

@ -244,6 +244,7 @@ from litellm.types.router import (
AllowedFailsPolicy,
AssistantsTypedDict,
AutoRouterCapabilityLimit,
CallerModelGrants,
ConsumedRequestTagsStamp,
CredentialLiteLLMParams,
CustomRoutingStrategyBase,
@ -12723,9 +12724,7 @@ class Router:
deployments: Sequence[Mapping[str, object]],
user_api_key_auth: object,
) -> list[Mapping[str, object]]:
from litellm.proxy._types import UserAPIKeyAuth
if not deployments or not isinstance(user_api_key_auth, UserAPIKeyAuth):
if not deployments or not isinstance(user_api_key_auth, CallerModelGrants):
return list(deployments)
model_group_ids: Final = frozenset(self.get_model_ids(model_name=model))

View file

@ -1065,6 +1065,17 @@ class ConsumedRequestTagsStamp:
tags: tuple[str, ...]
@runtime_checkable
class CallerModelGrants(Protocol):
"""Key and team model grants a proxy request carries as ``user_api_key_auth`` in its metadata."""
@property
def models(self) -> Sequence[object]: ...
@property
def team_models(self) -> Sequence[object]: ...
@runtime_checkable
class PreRoutingStrategy(Protocol):
"""Structural interface shared by the auto / complexity / adaptive / quality routers."""

View file

@ -20,6 +20,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
_safe_get_request_parsed_body,
_safe_get_request_query_params,
_safe_set_request_parsed_body,
check_file_size_under_limit,
coerce_numeric_form_fields,
get_form_data,
get_request_body,
@ -1203,3 +1204,49 @@ class TestCoerceNumericFormFields:
numeric_fields=self.numeric_fields,
)
assert result == {"n": 3, "temperature": None, "image": buffer}
def test_check_file_size_under_limit_uses_the_granted_deployment(monkeypatch):
from starlette.datastructures import UploadFile
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
router = litellm.Router(
model_list=[
{
"model_name": "whisper",
"litellm_params": {"model": "openai/whisper-1", "api_key": "key1"},
"model_info": {"id": "openai-whisper-id"},
},
{
"model_name": "whisper",
"litellm_params": {
"model": "azure/whisper",
"api_key": "key2",
"api_base": "https://x",
"max_file_size_mb": 0.000001,
},
"model_info": {"id": "azure-whisper-id"},
},
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "premium_user", True)
upload = UploadFile(filename="audio.mp3", file=io.BytesIO(b"\x00" * 16), size=16)
request_data = {"model": "whisper"}
assert check_file_size_under_limit(request_data, upload, ["whisper"]) is True
assert (
check_file_size_under_limit(
request_data, upload, ["whisper"], user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["whisper"])
)
is True
)
with pytest.raises(ProxyException, match="File size is too large"):
check_file_size_under_limit(
request_data,
upload,
["whisper"],
user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["azure-whisper-id"]),
)

View file

@ -302,6 +302,36 @@ def test_get_model_by_id_not_found(client, auth_as, patched_models, path):
assert "not found" in response.text.lower()
@pytest.mark.parametrize("path", ["/v1/models/gpt-4", "/models/gpt-4"])
def test_get_model_by_id_follows_the_granted_deployment(client, auth_as, patched_models, monkeypatch, path):
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
monkeypatch.setattr(litellm, "get_llm_provider", get_llm_provider)
router = litellm.Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "openai/gpt-4", "api_key": "key1"},
"model_info": {"id": "openai-gpt-4-id"},
},
{
"model_name": "gpt-4",
"litellm_params": {"model": "azure/gpt-4", "api_key": "key2", "api_base": "https://x"},
"model_info": {"id": "azure-gpt-4-id"},
},
]
)
monkeypatch.setattr(proxy_server, "llm_router", router)
with auth_as(models=["azure-gpt-4-id"]):
granted = client.get(path)
with auth_as(models=["gpt-4"]):
broad = client.get(path)
assert granted.status_code == 200 and granted.json()["owned_by"] == "azure"
assert broad.status_code == 200 and broad.json()["owned_by"] == "openai"
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
def test_anthropic_format_returns_public_team_model_name(
client, auth_as, patched_models, monkeypatch, params

View file

@ -1553,7 +1553,7 @@ async def test_retrieve_model_by_public_name_returns_200(monkeypatch):
assert resp.get("max_output_tokens") == 4096
# lookup happened by the internal routing key, not the public name
router.get_deployment_by_model_group_name.assert_called_once_with(
"model_name_team-abc-123_4a6b8"
"model_name_team-abc-123_4a6b8", user_api_key_auth=key
)

View file

@ -5898,7 +5898,7 @@ def test_resolve_provider_from_deployment_falls_back_to_pre_alias():
deployment.litellm_params.model = "bedrock/anthropic.claude-sonnet-4-6"
deployment.litellm_params.custom_llm_provider = None
def lookup(model_group_name):
def lookup(model_group_name, user_api_key_auth=None):
if model_group_name == "pre-alias-name":
return deployment
return None
@ -5909,6 +5909,31 @@ def test_resolve_provider_from_deployment_falls_back_to_pre_alias():
assert result == "bedrock"
def test_resolve_provider_from_deployment_follows_the_granted_deployment():
router = litellm.Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "openai/gpt-4", "api_key": "key1"},
"model_info": {"id": "openai-gpt-4-id"},
},
{
"model_name": "gpt-4",
"litellm_params": {"model": "azure/gpt-4", "api_key": "key2", "api_base": "https://x"},
"model_info": {"id": "azure-gpt-4-id"},
},
]
)
assert _resolve_provider_from_deployment(router, "gpt-4") == "openai"
assert (
_resolve_provider_from_deployment(
router, "gpt-4", user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["azure-gpt-4-id"])
)
== "azure"
)
def test_apply_overrides_multi_provider_default_picks_correct_provider(
setup_test_credentials,
):

View file

@ -1,5 +1,6 @@
import asyncio
import copy
import dataclasses
import functools
import gc
import json
@ -16888,6 +16889,24 @@ def test_deployment_id_grant_restricts_routing_to_granted_deployment():
assert seen == {"from-azure"}
def test_deployment_id_grant_filter_does_not_import_proxy_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:
@dataclasses.dataclass(frozen=True, slots=True)
class Grants:
models: tuple[str, ...]
team_models: tuple[str, ...]
router = _same_name_router_for_deployment_id_grants()
monkeypatch.setitem(sys.modules, "fastapi", None)
monkeypatch.delitem(sys.modules, "litellm.proxy._types", raising=False)
_model, deployments = router._common_checks_available_deployment(
model="gpt-4",
request_kwargs={"metadata": {"user_api_key_auth": Grants(models=("azure-gpt-4-id",), team_models=())}},
)
assert [d["model_info"]["id"] for d in deployments] == ["azure-gpt-4-id"]
def test_public_name_grant_is_not_narrowed_by_deployment_id_grant():
from litellm.proxy._types import UserAPIKeyAuth