From 8ce2a7a149cfd443e25c214dc2a342ce2b50efa8 Mon Sep 17 00:00:00 2001 From: ryan Date: Sat, 19 Sep 2026 01:25:47 +0000 Subject: [PATCH] fix(router): keep deployment id grants out of the sdk proxy import path Check caller grants through a runtime protocol instead of importing UserAPIKeyAuth into router.py, and pass caller auth through the remaining public-name lookups (model info, token-array embeddings, file size limits, provider resolution) so a deployment id grant resolves the granted deployment rather than its first sibling Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/common_utils/http_parsing_utils.py | 5 +- litellm/proxy/litellm_pre_call_utils.py | 6 ++- litellm/proxy/proxy_server.py | 9 +++- litellm/router.py | 5 +- litellm/types/router.py | 11 +++++ .../common_utils/test_http_parsing_utils.py | 47 +++++++++++++++++++ .../proxy/proxy_server/test_routes_models.py | 30 ++++++++++++ .../test_team_model_name_translation.py | 2 +- .../proxy/test_litellm_pre_call_utils.py | 27 ++++++++++- tests/test_litellm/test_router.py | 19 ++++++++ 10 files changed, 151 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1c17c46e5af..29b437a3f22 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -14,7 +14,7 @@ from litellm.constants import ( CLIENT_REQUESTED_MODEL_SCOPE_KEY, MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB, ) -from litellm.proxy._types import ProxyException +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( get_metadata_variable_name_from_kwargs, ) @@ -329,6 +329,7 @@ def check_file_size_under_limit( request_data: dict, file: UploadFile, router_model_names: Collection[str], + user_api_key_dict: UserAPIKeyAuth | None = None, ) -> bool: """ Check if any files passed in request are under max_file_size_mb @@ -353,7 +354,7 @@ def check_file_size_under_limit( if llm_router is not None and request_data["model"] in router_model_names: try: deployment: Final[Deployment | None] = llm_router.get_deployment_by_model_group_name( - model_group_name=request_data["model"] + model_group_name=request_data["model"], user_api_key_auth=user_api_key_dict ) if ( deployment diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index b03f1e4348c..f8665863450 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2692,6 +2692,7 @@ def _apply_credential_overrides_from_model_config( llm_router=llm_router, model_name=model_name, pre_alias_model_name=pre_alias_model_name, + user_api_key_dict=user_api_key_dict, ) credential_name: Final = _resolve_credential_from_model_config( @@ -2732,6 +2733,7 @@ def _resolve_provider_from_deployment( llm_router: Router, model_name: str, pre_alias_model_name: str | None = None, + user_api_key_dict: UserAPIKeyAuth | None = None, ) -> str | None: """ Resolve a provider hint from the deployment's litellm_params when the @@ -2747,7 +2749,9 @@ def _resolve_provider_from_deployment( for name in candidates: try: - deployment = llm_router.get_deployment_by_model_group_name(model_group_name=name) + deployment = llm_router.get_deployment_by_model_group_name( + model_group_name=name, user_api_key_auth=user_api_key_dict + ) except Exception: deployment = None if deployment is None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 66d33ad7102..8e19906db38 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11051,7 +11051,9 @@ async def model_info( if llm_router is None: raise HTTPException(status_code=500, detail="Router not initialized") - deployment: Final = llm_router.get_deployment_by_model_group_name(resolved_model_id) + deployment: Final = llm_router.get_deployment_by_model_group_name( + resolved_model_id, user_api_key_auth=user_api_key_dict + ) if deployment is None: raise HTTPException( status_code=404, @@ -11495,7 +11497,9 @@ async def embeddings( # check if provider accept list of tokens as input - e.g. for langchain integration if llm_router is not None and data.get("model") in router_model_names: # Use router's O(1) lookup instead of O(N) iteration through llm_model_list - deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=data["model"]) + deployment: Final = llm_router.get_deployment_by_model_group_name( + model_group_name=data["model"], user_api_key_auth=user_api_key_dict + ) if deployment is not None: litellm_params: Final = deployment.get("litellm_params", {}) or {} litellm_model: Final = litellm_params.get("model", "") @@ -11882,6 +11886,7 @@ async def audio_transcriptions( request_data=data, file=file, router_model_names=router_model_names, + user_api_key_dict=user_api_key_dict, ) file_content: Final = await file.read() diff --git a/litellm/router.py b/litellm/router.py index 55e185d91ed..b718d77dd0c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -244,6 +244,7 @@ from litellm.types.router import ( AllowedFailsPolicy, AssistantsTypedDict, AutoRouterCapabilityLimit, + CallerModelGrants, ConsumedRequestTagsStamp, CredentialLiteLLMParams, CustomRoutingStrategyBase, @@ -12723,9 +12724,7 @@ class Router: deployments: Sequence[Mapping[str, object]], user_api_key_auth: object, ) -> list[Mapping[str, object]]: - from litellm.proxy._types import UserAPIKeyAuth - - if not deployments or not isinstance(user_api_key_auth, UserAPIKeyAuth): + if not deployments or not isinstance(user_api_key_auth, CallerModelGrants): return list(deployments) model_group_ids: Final = frozenset(self.get_model_ids(model_name=model)) diff --git a/litellm/types/router.py b/litellm/types/router.py index adadb053ab2..f7dcf0f5aba 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1065,6 +1065,17 @@ class ConsumedRequestTagsStamp: tags: tuple[str, ...] +@runtime_checkable +class CallerModelGrants(Protocol): + """Key and team model grants a proxy request carries as ``user_api_key_auth`` in its metadata.""" + + @property + def models(self) -> Sequence[object]: ... + + @property + def team_models(self) -> Sequence[object]: ... + + @runtime_checkable class PreRoutingStrategy(Protocol): """Structural interface shared by the auto / complexity / adaptive / quality routers.""" diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 72cd7a218d3..9b9e058d4d1 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -20,6 +20,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( _safe_get_request_parsed_body, _safe_get_request_query_params, _safe_set_request_parsed_body, + check_file_size_under_limit, coerce_numeric_form_fields, get_form_data, get_request_body, @@ -1203,3 +1204,49 @@ class TestCoerceNumericFormFields: numeric_fields=self.numeric_fields, ) assert result == {"n": 3, "temperature": None, "image": buffer} + + +def test_check_file_size_under_limit_uses_the_granted_deployment(monkeypatch): + from starlette.datastructures import UploadFile + + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + router = litellm.Router( + model_list=[ + { + "model_name": "whisper", + "litellm_params": {"model": "openai/whisper-1", "api_key": "key1"}, + "model_info": {"id": "openai-whisper-id"}, + }, + { + "model_name": "whisper", + "litellm_params": { + "model": "azure/whisper", + "api_key": "key2", + "api_base": "https://x", + "max_file_size_mb": 0.000001, + }, + "model_info": {"id": "azure-whisper-id"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "premium_user", True) + upload = UploadFile(filename="audio.mp3", file=io.BytesIO(b"\x00" * 16), size=16) + request_data = {"model": "whisper"} + + assert check_file_size_under_limit(request_data, upload, ["whisper"]) is True + assert ( + check_file_size_under_limit( + request_data, upload, ["whisper"], user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["whisper"]) + ) + is True + ) + with pytest.raises(ProxyException, match="File size is too large"): + check_file_size_under_limit( + request_data, + upload, + ["whisper"], + user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["azure-whisper-id"]), + ) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_models.py b/tests/test_litellm/proxy/proxy_server/test_routes_models.py index d31d952a03e..d7257f1cc95 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_models.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_models.py @@ -302,6 +302,36 @@ def test_get_model_by_id_not_found(client, auth_as, patched_models, path): assert "not found" in response.text.lower() +@pytest.mark.parametrize("path", ["/v1/models/gpt-4", "/models/gpt-4"]) +def test_get_model_by_id_follows_the_granted_deployment(client, auth_as, patched_models, monkeypatch, path): + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + monkeypatch.setattr(litellm, "get_llm_provider", get_llm_provider) + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "openai/gpt-4", "api_key": "key1"}, + "model_info": {"id": "openai-gpt-4-id"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "azure/gpt-4", "api_key": "key2", "api_base": "https://x"}, + "model_info": {"id": "azure-gpt-4-id"}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + + with auth_as(models=["azure-gpt-4-id"]): + granted = client.get(path) + with auth_as(models=["gpt-4"]): + broad = client.get(path) + + assert granted.status_code == 200 and granted.json()["owned_by"] == "azure" + assert broad.status_code == 200 and broad.json()["owned_by"] == "openai" + + @pytest.mark.parametrize("params", [{}, {"scope": "expand"}]) def test_anthropic_format_returns_public_team_model_name( client, auth_as, patched_models, monkeypatch, params diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index 1d2e1682460..84abe130314 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -1553,7 +1553,7 @@ async def test_retrieve_model_by_public_name_returns_200(monkeypatch): assert resp.get("max_output_tokens") == 4096 # lookup happened by the internal routing key, not the public name router.get_deployment_by_model_group_name.assert_called_once_with( - "model_name_team-abc-123_4a6b8" + "model_name_team-abc-123_4a6b8", user_api_key_auth=key ) diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index f4490519554..a4b9c2a44a0 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -5898,7 +5898,7 @@ def test_resolve_provider_from_deployment_falls_back_to_pre_alias(): deployment.litellm_params.model = "bedrock/anthropic.claude-sonnet-4-6" deployment.litellm_params.custom_llm_provider = None - def lookup(model_group_name): + def lookup(model_group_name, user_api_key_auth=None): if model_group_name == "pre-alias-name": return deployment return None @@ -5909,6 +5909,31 @@ def test_resolve_provider_from_deployment_falls_back_to_pre_alias(): assert result == "bedrock" +def test_resolve_provider_from_deployment_follows_the_granted_deployment(): + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "openai/gpt-4", "api_key": "key1"}, + "model_info": {"id": "openai-gpt-4-id"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "azure/gpt-4", "api_key": "key2", "api_base": "https://x"}, + "model_info": {"id": "azure-gpt-4-id"}, + }, + ] + ) + + assert _resolve_provider_from_deployment(router, "gpt-4") == "openai" + assert ( + _resolve_provider_from_deployment( + router, "gpt-4", user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["azure-gpt-4-id"]) + ) + == "azure" + ) + + def test_apply_overrides_multi_provider_default_picks_correct_provider( setup_test_credentials, ): diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index ecce1adbf6a..024e6949f5b 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1,5 +1,6 @@ import asyncio import copy +import dataclasses import functools import gc import json @@ -16888,6 +16889,24 @@ def test_deployment_id_grant_restricts_routing_to_granted_deployment(): assert seen == {"from-azure"} +def test_deployment_id_grant_filter_does_not_import_proxy_dependencies(monkeypatch: pytest.MonkeyPatch) -> None: + @dataclasses.dataclass(frozen=True, slots=True) + class Grants: + models: tuple[str, ...] + team_models: tuple[str, ...] + + router = _same_name_router_for_deployment_id_grants() + monkeypatch.setitem(sys.modules, "fastapi", None) + monkeypatch.delitem(sys.modules, "litellm.proxy._types", raising=False) + + _model, deployments = router._common_checks_available_deployment( + model="gpt-4", + request_kwargs={"metadata": {"user_api_key_auth": Grants(models=("azure-gpt-4-id",), team_models=())}}, + ) + + assert [d["model_info"]["id"] for d in deployments] == ["azure-gpt-4-id"] + + def test_public_name_grant_is_not_narrowed_by_deployment_id_grant(): from litellm.proxy._types import UserAPIKeyAuth