mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(router): keep deployment id grants out of the sdk proxy import path
Check caller grants through a runtime protocol instead of importing UserAPIKeyAuth into router.py, and pass caller auth through the remaining public-name lookups (model info, token-array embeddings, file size limits, provider resolution) so a deployment id grant resolves the granted deployment rather than its first sibling Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
796576d8df
commit
8ce2a7a149
10 changed files with 151 additions and 10 deletions
|
|
@ -14,7 +14,7 @@ from litellm.constants import (
|
|||
CLIENT_REQUESTED_MODEL_SCOPE_KEY,
|
||||
MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB,
|
||||
)
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy._types import ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.callback_utils import (
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
)
|
||||
|
|
@ -329,6 +329,7 @@ def check_file_size_under_limit(
|
|||
request_data: dict,
|
||||
file: UploadFile,
|
||||
router_model_names: Collection[str],
|
||||
user_api_key_dict: UserAPIKeyAuth | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if any files passed in request are under max_file_size_mb
|
||||
|
|
@ -353,7 +354,7 @@ def check_file_size_under_limit(
|
|||
if llm_router is not None and request_data["model"] in router_model_names:
|
||||
try:
|
||||
deployment: Final[Deployment | None] = llm_router.get_deployment_by_model_group_name(
|
||||
model_group_name=request_data["model"]
|
||||
model_group_name=request_data["model"], user_api_key_auth=user_api_key_dict
|
||||
)
|
||||
if (
|
||||
deployment
|
||||
|
|
|
|||
|
|
@ -2692,6 +2692,7 @@ def _apply_credential_overrides_from_model_config(
|
|||
llm_router=llm_router,
|
||||
model_name=model_name,
|
||||
pre_alias_model_name=pre_alias_model_name,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
credential_name: Final = _resolve_credential_from_model_config(
|
||||
|
|
@ -2732,6 +2733,7 @@ def _resolve_provider_from_deployment(
|
|||
llm_router: Router,
|
||||
model_name: str,
|
||||
pre_alias_model_name: str | None = None,
|
||||
user_api_key_dict: UserAPIKeyAuth | None = None,
|
||||
) -> str | None:
|
||||
"""
|
||||
Resolve a provider hint from the deployment's litellm_params when the
|
||||
|
|
@ -2747,7 +2749,9 @@ def _resolve_provider_from_deployment(
|
|||
|
||||
for name in candidates:
|
||||
try:
|
||||
deployment = llm_router.get_deployment_by_model_group_name(model_group_name=name)
|
||||
deployment = llm_router.get_deployment_by_model_group_name(
|
||||
model_group_name=name, user_api_key_auth=user_api_key_dict
|
||||
)
|
||||
except Exception:
|
||||
deployment = None
|
||||
if deployment is None:
|
||||
|
|
|
|||
|
|
@ -11051,7 +11051,9 @@ async def model_info(
|
|||
if llm_router is None:
|
||||
raise HTTPException(status_code=500, detail="Router not initialized")
|
||||
|
||||
deployment: Final = llm_router.get_deployment_by_model_group_name(resolved_model_id)
|
||||
deployment: Final = llm_router.get_deployment_by_model_group_name(
|
||||
resolved_model_id, user_api_key_auth=user_api_key_dict
|
||||
)
|
||||
if deployment is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
|
|
@ -11495,7 +11497,9 @@ async def embeddings(
|
|||
# check if provider accept list of tokens as input - e.g. for langchain integration
|
||||
if llm_router is not None and data.get("model") in router_model_names:
|
||||
# Use router's O(1) lookup instead of O(N) iteration through llm_model_list
|
||||
deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=data["model"])
|
||||
deployment: Final = llm_router.get_deployment_by_model_group_name(
|
||||
model_group_name=data["model"], user_api_key_auth=user_api_key_dict
|
||||
)
|
||||
if deployment is not None:
|
||||
litellm_params: Final = deployment.get("litellm_params", {}) or {}
|
||||
litellm_model: Final = litellm_params.get("model", "")
|
||||
|
|
@ -11882,6 +11886,7 @@ async def audio_transcriptions(
|
|||
request_data=data,
|
||||
file=file,
|
||||
router_model_names=router_model_names,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
file_content: Final = await file.read()
|
||||
|
|
|
|||
|
|
@ -244,6 +244,7 @@ from litellm.types.router import (
|
|||
AllowedFailsPolicy,
|
||||
AssistantsTypedDict,
|
||||
AutoRouterCapabilityLimit,
|
||||
CallerModelGrants,
|
||||
ConsumedRequestTagsStamp,
|
||||
CredentialLiteLLMParams,
|
||||
CustomRoutingStrategyBase,
|
||||
|
|
@ -12723,9 +12724,7 @@ class Router:
|
|||
deployments: Sequence[Mapping[str, object]],
|
||||
user_api_key_auth: object,
|
||||
) -> list[Mapping[str, object]]:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
if not deployments or not isinstance(user_api_key_auth, UserAPIKeyAuth):
|
||||
if not deployments or not isinstance(user_api_key_auth, CallerModelGrants):
|
||||
return list(deployments)
|
||||
|
||||
model_group_ids: Final = frozenset(self.get_model_ids(model_name=model))
|
||||
|
|
|
|||
|
|
@ -1065,6 +1065,17 @@ class ConsumedRequestTagsStamp:
|
|||
tags: tuple[str, ...]
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class CallerModelGrants(Protocol):
|
||||
"""Key and team model grants a proxy request carries as ``user_api_key_auth`` in its metadata."""
|
||||
|
||||
@property
|
||||
def models(self) -> Sequence[object]: ...
|
||||
|
||||
@property
|
||||
def team_models(self) -> Sequence[object]: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class PreRoutingStrategy(Protocol):
|
||||
"""Structural interface shared by the auto / complexity / adaptive / quality routers."""
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
|
|||
_safe_get_request_parsed_body,
|
||||
_safe_get_request_query_params,
|
||||
_safe_set_request_parsed_body,
|
||||
check_file_size_under_limit,
|
||||
coerce_numeric_form_fields,
|
||||
get_form_data,
|
||||
get_request_body,
|
||||
|
|
@ -1203,3 +1204,49 @@ class TestCoerceNumericFormFields:
|
|||
numeric_fields=self.numeric_fields,
|
||||
)
|
||||
assert result == {"n": 3, "temperature": None, "image": buffer}
|
||||
|
||||
|
||||
def test_check_file_size_under_limit_uses_the_granted_deployment(monkeypatch):
|
||||
from starlette.datastructures import UploadFile
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "whisper",
|
||||
"litellm_params": {"model": "openai/whisper-1", "api_key": "key1"},
|
||||
"model_info": {"id": "openai-whisper-id"},
|
||||
},
|
||||
{
|
||||
"model_name": "whisper",
|
||||
"litellm_params": {
|
||||
"model": "azure/whisper",
|
||||
"api_key": "key2",
|
||||
"api_base": "https://x",
|
||||
"max_file_size_mb": 0.000001,
|
||||
},
|
||||
"model_info": {"id": "azure-whisper-id"},
|
||||
},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "premium_user", True)
|
||||
upload = UploadFile(filename="audio.mp3", file=io.BytesIO(b"\x00" * 16), size=16)
|
||||
request_data = {"model": "whisper"}
|
||||
|
||||
assert check_file_size_under_limit(request_data, upload, ["whisper"]) is True
|
||||
assert (
|
||||
check_file_size_under_limit(
|
||||
request_data, upload, ["whisper"], user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["whisper"])
|
||||
)
|
||||
is True
|
||||
)
|
||||
with pytest.raises(ProxyException, match="File size is too large"):
|
||||
check_file_size_under_limit(
|
||||
request_data,
|
||||
upload,
|
||||
["whisper"],
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["azure-whisper-id"]),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -302,6 +302,36 @@ def test_get_model_by_id_not_found(client, auth_as, patched_models, path):
|
|||
assert "not found" in response.text.lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/models/gpt-4", "/models/gpt-4"])
|
||||
def test_get_model_by_id_follows_the_granted_deployment(client, auth_as, patched_models, monkeypatch, path):
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
monkeypatch.setattr(litellm, "get_llm_provider", get_llm_provider)
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "openai/gpt-4", "api_key": "key1"},
|
||||
"model_info": {"id": "openai-gpt-4-id"},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "azure/gpt-4", "api_key": "key2", "api_base": "https://x"},
|
||||
"model_info": {"id": "azure-gpt-4-id"},
|
||||
},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
|
||||
with auth_as(models=["azure-gpt-4-id"]):
|
||||
granted = client.get(path)
|
||||
with auth_as(models=["gpt-4"]):
|
||||
broad = client.get(path)
|
||||
|
||||
assert granted.status_code == 200 and granted.json()["owned_by"] == "azure"
|
||||
assert broad.status_code == 200 and broad.json()["owned_by"] == "openai"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("params", [{}, {"scope": "expand"}])
|
||||
def test_anthropic_format_returns_public_team_model_name(
|
||||
client, auth_as, patched_models, monkeypatch, params
|
||||
|
|
|
|||
|
|
@ -1553,7 +1553,7 @@ async def test_retrieve_model_by_public_name_returns_200(monkeypatch):
|
|||
assert resp.get("max_output_tokens") == 4096
|
||||
# lookup happened by the internal routing key, not the public name
|
||||
router.get_deployment_by_model_group_name.assert_called_once_with(
|
||||
"model_name_team-abc-123_4a6b8"
|
||||
"model_name_team-abc-123_4a6b8", user_api_key_auth=key
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5898,7 +5898,7 @@ def test_resolve_provider_from_deployment_falls_back_to_pre_alias():
|
|||
deployment.litellm_params.model = "bedrock/anthropic.claude-sonnet-4-6"
|
||||
deployment.litellm_params.custom_llm_provider = None
|
||||
|
||||
def lookup(model_group_name):
|
||||
def lookup(model_group_name, user_api_key_auth=None):
|
||||
if model_group_name == "pre-alias-name":
|
||||
return deployment
|
||||
return None
|
||||
|
|
@ -5909,6 +5909,31 @@ def test_resolve_provider_from_deployment_falls_back_to_pre_alias():
|
|||
assert result == "bedrock"
|
||||
|
||||
|
||||
def test_resolve_provider_from_deployment_follows_the_granted_deployment():
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "openai/gpt-4", "api_key": "key1"},
|
||||
"model_info": {"id": "openai-gpt-4-id"},
|
||||
},
|
||||
{
|
||||
"model_name": "gpt-4",
|
||||
"litellm_params": {"model": "azure/gpt-4", "api_key": "key2", "api_base": "https://x"},
|
||||
"model_info": {"id": "azure-gpt-4-id"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert _resolve_provider_from_deployment(router, "gpt-4") == "openai"
|
||||
assert (
|
||||
_resolve_provider_from_deployment(
|
||||
router, "gpt-4", user_api_key_dict=UserAPIKeyAuth(api_key="k", models=["azure-gpt-4-id"])
|
||||
)
|
||||
== "azure"
|
||||
)
|
||||
|
||||
|
||||
def test_apply_overrides_multi_provider_default_picks_correct_provider(
|
||||
setup_test_credentials,
|
||||
):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import dataclasses
|
||||
import functools
|
||||
import gc
|
||||
import json
|
||||
|
|
@ -16888,6 +16889,24 @@ def test_deployment_id_grant_restricts_routing_to_granted_deployment():
|
|||
assert seen == {"from-azure"}
|
||||
|
||||
|
||||
def test_deployment_id_grant_filter_does_not_import_proxy_dependencies(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class Grants:
|
||||
models: tuple[str, ...]
|
||||
team_models: tuple[str, ...]
|
||||
|
||||
router = _same_name_router_for_deployment_id_grants()
|
||||
monkeypatch.setitem(sys.modules, "fastapi", None)
|
||||
monkeypatch.delitem(sys.modules, "litellm.proxy._types", raising=False)
|
||||
|
||||
_model, deployments = router._common_checks_available_deployment(
|
||||
model="gpt-4",
|
||||
request_kwargs={"metadata": {"user_api_key_auth": Grants(models=("azure-gpt-4-id",), team_models=())}},
|
||||
)
|
||||
|
||||
assert [d["model_info"]["id"] for d in deployments] == ["azure-gpt-4-id"]
|
||||
|
||||
|
||||
def test_public_name_grant_is_not_narrowed_by_deployment_id_grant():
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue