From 9e3098529046fd16b5050b0557ff8496bf0229c8 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Wed, 17 Jun 2026 10:32:52 -0700 Subject: [PATCH] fix(proxy): resolve list files credentials from team BYOK deployments (#30495) * fix(proxy): resolve list files credentials from team BYOK deployments GET /v1/files without target_model_names now prefers the team's own deployment (model_info.team_id) over shared global provider keys, so JWT team auth lists files against the correct upstream account. Co-authored-by: Cursor * fix(proxy): scope list files credential lookup to team allowlist Remove the unrestricted deployment scan that could leak global provider keys to teams without access, normalize all-proxy-models to the team-scoped model list, and fix TID251 violations by using dict instead of Dict/Any. Co-authored-by: Cursor --------- Co-authored-by: Cursor (cherry picked from commit 6c8b60d50d9985e5e56e09e1081a63d445598e17) --- .../openai_files_endpoints/common_utils.py | 88 +++++ .../openai_files_endpoints/files_endpoints.py | 25 +- .../test_files_endpoint.py | 326 ++++++++++++++++++ 3 files changed, 436 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index cc0d06e4f40..694924f2b90 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -10,6 +10,8 @@ from litellm.types.utils import SpecialEnums if TYPE_CHECKING: from fastapi import Request + from litellm.router import Router + def _is_base64_encoded_unified_file_id(b64_uid: str) -> Union[str, Literal[False]]: # Ensure b64_uid is a string and not a mock object @@ -296,6 +298,92 @@ def get_credentials_for_model( return credentials +def get_team_provider_credentials( + llm_router: Optional["Router"], + team_models: List[str], + custom_llm_provider: str, + team_id: Optional[str] = None, +) -> Optional[dict]: + """ + Resolve upstream credentials for a provider-scoped file operation + (e.g. GET /v1/files), which doesn't pin a model. + + Priority: + 1. The team's own (BYOK) deployment for this provider — a deployment whose + ``model_info.team_id`` matches ``team_id``. This keeps team-scoped listings + on the team's own provider account/key instead of a shared global one. + 2. Fallback: any deployment the team is granted access to for this provider, + expanding wildcard routes and the all-proxy-models sentinel. + + Credential lookup is always scoped to the team's allowlist, so a team can + never resolve a provider key for a deployment it isn't authorized to use. + Returns None when the router is unavailable or no authorized deployment + matches, so the caller can fall back to default credential resolution. + """ + if llm_router is None: + return None + + def _provider_credentials(model_id: str) -> Optional[dict]: + credentials = llm_router.get_deployment_credentials_with_provider( + model_id=model_id + ) + if ( + credentials is not None + and credentials.get("custom_llm_provider") == custom_llm_provider + ): + return credentials + return None + + # 1. Prefer the team's own BYOK deployment, matched by model_info.team_id. + if team_id is not None: + for deployment in llm_router.model_list or []: + model_info = deployment.get("model_info") or {} + if model_info.get("team_id") != team_id: + continue + deployment_id = model_info.get("id") + if deployment_id is None: + continue + credentials = _provider_credentials(deployment_id) + if credentials is not None: + return credentials + + # 2. Fall back to deployments the team is allowed to access. The + # all-proxy-models sentinel isn't expanded by get_complete_model_list, so + # normalize it to an empty allowlist, which defers to the team-scoped + # proxy model list. A team with a restricted allowlist (e.g. anthropic + # only) therefore never resolves another provider's key. + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.model_checks import get_complete_model_list + + grants_all_models = SpecialModelNames.all_proxy_models.value in team_models + effective_team_models = [] if grants_all_models else team_models + + proxy_model_list = llm_router.get_model_names(team_id=team_id) + model_access_groups = llm_router.get_model_access_groups() + models_to_try = list( + dict.fromkeys( + get_complete_model_list( + key_models=[], + team_models=effective_team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=True, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=True, + team_id=team_id, + ) + ) + ) + for model_name in models_to_try: + credentials = _provider_credentials(model_name) + if credentials is not None: + return credentials + + return None + + def prepare_data_with_credentials( data: dict, credentials: dict, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 378cbbda89c..57b6d111e0e 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -51,6 +51,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( encode_file_id_with_model, extract_file_creation_params, get_credentials_for_model, + get_team_provider_credentials, handle_model_based_routing, prepare_data_with_credentials, ) @@ -1344,14 +1345,20 @@ async def list_files( status_code=400, detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']", ) - ## Use router to list fine-tuning jobs for that model if llm_router is None: raise HTTPException( status_code=500, detail="LLM Router not initialized. Ensure models added to proxy.", ) - data["model"] = target_model_names_list[0] - response = await llm_router.afile_list( + credentials = get_credentials_for_model( + llm_router=llm_router, + model_id=target_model_names_list[0], + operation_context="file list", + ) + prepare_data_with_credentials(data=data, credentials=credentials) + response = await litellm.afile_list( + custom_llm_provider=credentials["custom_llm_provider"], + purpose=purpose, **data, ) else: @@ -1363,6 +1370,18 @@ async def list_files( or "openai" ) + # No model/target_model_names pinned: resolve upstream credentials from + # the team's deployment for this provider so the call is authenticated + # against the team's own account (e.g. the team's openai deployment). + team_credentials = get_team_provider_credentials( + llm_router=llm_router, + team_models=user_api_key_dict.team_models or [], + custom_llm_provider=custom_llm_provider, + team_id=user_api_key_dict.team_id, + ) + if team_credentials is not None: + prepare_data_with_credentials(data=data, credentials=team_credentials) + response = await litellm.afile_list( custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore ) diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 5fc36b71f2b..007c04e0aff 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -1873,3 +1873,329 @@ def test_get_file_content_non_openai_provider_skips_streaming_handler( assert "stream" not in captured_kwargs mock_streaming_response.assert_not_awaited() proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_resolves_wildcard_deployment_credentials( + mocker: MockerFixture, monkeypatch +): + """ + GET /v1/files?target_model_names= must resolve the upstream api_key + from the matching (wildcard) deployment. Regression for the path routing + through llm_router.afile_list(model=...), which reached OpenAI without an + api_key and failed with "api_key client option must be set". + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + wildcard_router = Router( + model_list=[ + { + "model_name": "*", + "litellm_params": { + "model": "openai/*", + "api_key": "wildcard-openai-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="test-user", + ) + + try: + response = client.get( + "/v1/files?target_model_names=gpt-4o", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "wildcard-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_without_target_model_names_uses_team_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + Plain GET /v1/files (no target_model_names) must resolve the upstream openai + api_key from the team's openai deployment instead of falling through to a + keyless OpenAI client. Regression for "api_key client option must be set". + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + wildcard_router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "team-openai-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=["openai/*"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_restricted_team_does_not_leak_global_openai_credentials( + mocker: MockerFixture, monkeypatch +): + """ + A team whose allowlist only grants anthropic must NOT resolve a global + openai deployment's api_key for plain GET /v1/files. Regression for the + last-resort scan that ignored team access control. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "global-openai-key", + }, + }, + { + "model_name": "claude-opus-4-6", + "litellm_params": { + "model": "anthropic/claude-opus-4-6", + "api_key": "anthropic-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="anthropic-only-team", + team_models=["claude-opus-4-6"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") != "global-openai-key" + + +def test_list_files_prefers_team_byok_over_global_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + When a team has its own BYOK openai deployment (model_info.team_id set), plain + GET /v1/files must use the team's key, not a shared/global openai deployment. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "global-openai-key", + }, + }, + { + "model_name": "team-gpt-4o", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "team-byok-openai-key", + }, + "model_info": { + "id": "team-byok-deployment-id", + "team_id": "test-team", + "team_public_model_name": "team-gpt-4o", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=["team-gpt-4o"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-byok-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_with_all_proxy_models_team_uses_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + Teams with all-proxy-models (or empty models) must still resolve openai + credentials for plain GET /v1/files. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles, SpecialModelNames + + wildcard_router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "team-openai-key", + }, + }, + { + "model_name": "claude-opus-4-6", + "litellm_params": { + "model": "anthropic/claude-opus-4-6", + "api_key": "anthropic-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=[SpecialModelNames.all_proxy_models.value], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called()