diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 48cbc0ab8ff..184f0868e93 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -306,6 +306,7 @@ def _is_xai_batch_output( class _ManagedFileRow(Protocol): unified_file_id: str file_object: OpenAIFileObject + model_mappings: Mapping[str, str] flat_model_file_ids: Sequence[str] storage_backend: Optional[str] storage_url: Optional[str] @@ -2260,14 +2261,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): user_api_key_dict: UserAPIKeyAuth, limit: Optional[int] = None, after: Optional[str] = None, + model_mapping_keys: Optional[frozenset[str]] = None, + before: Optional[str] = None, **data: Dict, ) -> FileListPage: """List the managed files the caller owns, newest first. Pagination is keyset based on ``unified_file_id`` so a key that owns every file on the proxy still reads one bounded page at a time. + ``after`` pages toward older files and ``before`` toward newer ones, + each page still ordered newest first. ``purpose`` is applied after parsing, because the managed file table - keeps it inside the ``file_object`` blob instead of a column, and rows + keeps it inside the ``file_object`` blob instead of a column, and + ``model_mapping_keys`` keeps only files mapped to one of those + deployment ids, or model names for storage-backed files, read from + ``model_mappings``, a JSON column. Rows whose blob will not parse drop out there too, so a chunk of rows can yield fewer matches than the page holds. Successive chunks are read until the page is full or the caller's rows run out, which keeps @@ -2287,30 +2295,44 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if owner_filter is None: return FileListPage.model_validate(build_list_page([])) - if after: + if after and before: + raise ProxyException( + message="Pass either 'after' (or 'after_id') or 'before_id' when listing files, not both.", + type="invalid_request_error", + param="before_id", + code=400, + openai_code="invalid_value", + ) + cursor_param: Final = "before_id" if before else "after" + cursor: Final = before or after + if cursor: cursor_row = await _managed_file_table(self.prisma_client).find_first( - where={**owner_filter, "unified_file_id": after} + where={**owner_filter, "unified_file_id": cursor} ) if cursor_row is None: raise ProxyException( - message=f"Invalid 'after' cursor: no file found with id '{after}'.", + message=f"Invalid '{cursor_param}' cursor: no file found with id '{cursor}'.", type="invalid_request_error", - param="after", + param=cursor_param, code=400, openai_code="invalid_value", ) + if model_mapping_keys is not None and not model_mapping_keys: + return FileListPage.model_validate(build_list_page([])) + page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT) matches: Final[List[OpenAIFileObject]] = [] - cursor_id = after + cursor_id = cursor chunk_size = page_size + 1 + direction: Final = "asc" if before else "desc" while len(matches) <= page_size: cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {} chunk = await _managed_file_table(self.prisma_client).find_many( where=owner_filter, take=chunk_size, - order=[{"created_at": "desc"}, {"unified_file_id": "desc"}], + order=[{"created_at": direction}, {"unified_file_id": direction}], **cursor_args, ) matches.extend( @@ -2318,13 +2340,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): for row in chunk if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None and (purpose is None or parsed_file_object.purpose == purpose) + and (model_mapping_keys is None or not model_mapping_keys.isdisjoint(row.model_mappings)) ) if len(chunk) < chunk_size: break cursor_id = chunk[-1].unified_file_id chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE) - return FileListPage.model_validate(build_list_page(matches[:page_size], has_more=len(matches) > page_size)) + page: Final = matches[:page_size] + return FileListPage.model_validate( + build_list_page(page[::-1] if before else page, has_more=len(matches) > page_size) + ) def _is_batch_polling_enabled(self) -> bool: """ diff --git a/litellm/constants.py b/litellm/constants.py index 2901b7d8948..d1038e8d394 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -129,6 +129,7 @@ MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024 S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY" S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY" MAX_FILE_LIST_LIMIT: Final = 10000 +ANTHROPIC_DEFAULT_FILE_LIST_LIMIT: Final = 20 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1) GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10 diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index 4346e214dc8..6660873e8b1 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -274,6 +274,8 @@ class BaseFileEndpoints(ABC): user_api_key_dict: UserAPIKeyAuth, limit: int | None = None, after: str | None = None, + model_mapping_keys: frozenset[str] | None = None, + before: str | None = None, **data: dict, ) -> FileListPage: pass diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index d2ceefffe47..dbc16d4c828 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -6,9 +6,11 @@ ###################################################################### import asyncio +import itertools import traceback -from collections.abc import Mapping, Sequence -from typing import Any, BinaryIO, Final, TypedDict, cast, get_args +from collections.abc import Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, BinaryIO, Final, NoReturn, TypedDict, cast, get_args import httpx from fastapi import ( @@ -28,6 +30,7 @@ from typing_extensions import ReadOnly import litellm from litellm import CreateFileRequest, get_secret_str from litellm._logging import verbose_proxy_logger +from litellm.constants import ANTHROPIC_DEFAULT_FILE_LIST_LIMIT from litellm.litellm_core_utils.cloud_storage_security import ( is_managed_cloud_storage_uri, ) @@ -112,6 +115,9 @@ from litellm.types.llms.openai import ( OpenAIFileObject, OpenAIFilesPurpose, ) +from litellm.types.proxy.auth.user_api_key_auth import user_api_key_has_admin_view +from litellm.types.router import Deployment +from litellm.types.utils import LlmProviders router: Final = APIRouter() @@ -1673,6 +1679,139 @@ def _as_file_list_page(response: object) -> object: return FileListPage.model_validate(page) +async def _list_owned_managed_files( + managed_files_obj: BaseFileEndpoints, + user_api_key_dict: UserAPIKeyAuth, + purpose: str | None, + limit: int | None, + after: str | None, + before: str | None, + model_mapping_keys: frozenset[str] | None, +) -> FileListPage: + return await managed_files_obj.afile_list( + purpose=purpose, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + user_api_key_dict=user_api_key_dict, + limit=limit, + after=after, + before=before, + model_mapping_keys=model_mapping_keys, + ) + + +@dataclass(frozen=True, slots=True) +class _UnscopedFileList: + status_code: int + detail: str + + +def _raise_unscoped_file_list(failure: _UnscopedFileList) -> NoReturn: + raise HTTPException(status_code=failure.status_code, detail=failure.detail) + + +def _single_target_model_name(target_model_names: str) -> str | _UnscopedFileList: + target_model_names_list: Final = target_model_names.split(",") + if len(target_model_names_list) != 1: + return _UnscopedFileList( + status_code=400, + detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']", + ) + return target_model_names_list[0] + + +async def _requested_custom_llm_provider(request: Request, provider: str | None) -> str | None: + return ( + provider + or get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) + or await get_custom_llm_provider_from_request_body(request=request) + ) + + +def _default_file_list_limit(custom_llm_provider: str | None) -> int | None: + return ANTHROPIC_DEFAULT_FILE_LIST_LIMIT if custom_llm_provider == LlmProviders.ANTHROPIC.value else None + + +def _deployment_model_mapping_keys(deployment: Deployment) -> frozenset[str]: + return frozenset(key for key in (deployment.model_name, deployment.model_info.id) if key is not None) + + +def _model_mapping_keys(deployments: Iterable[Deployment]) -> frozenset[str]: + return frozenset(itertools.chain.from_iterable(map(_deployment_model_mapping_keys, deployments))) + + +def _model_mapping_keys_serving_model(llm_router: Router, model: str) -> frozenset[str]: + deployment: Final = llm_router.get_deployment(model_id=model) if llm_router.has_model_id(model) else None + if deployment is not None: + return _deployment_model_mapping_keys(deployment) + return _model_mapping_keys( + Deployment.model_validate(listed) for listed in llm_router.get_model_list(model_name=model) or () + ) + + +def _resolved_deployment_provider(deployment: Deployment) -> str | None: + try: + return litellm.get_llm_provider( + model=deployment.litellm_params.model, + custom_llm_provider=deployment.litellm_params.custom_llm_provider, + )[1] + except litellm.BadRequestError: + return None + + +def _deployments_on_provider(llm_router: Router | None, custom_llm_provider: str) -> Iterator[Deployment]: + if llm_router is None: + return iter(()) + return ( + deployment + for deployment_id in llm_router.get_model_ids() + if (deployment := llm_router.get_deployment(model_id=deployment_id)) is not None + and _resolved_deployment_provider(deployment) == custom_llm_provider + ) + + +def _model_mapping_keys_on_provider( + llm_router: Router | None, custom_llm_provider: str +) -> frozenset[str] | _UnscopedFileList: + from litellm.proxy.pass_through_endpoints.managed_id_rewriter import passthrough_file_model_id + + if custom_llm_provider not in litellm.provider_list: + return _UnscopedFileList(status_code=400, detail=f"Unknown provider '{custom_llm_provider}' for listing files.") + passthrough_model_id: Final = passthrough_file_model_id(custom_llm_provider) + return _model_mapping_keys(_deployments_on_provider(llm_router, custom_llm_provider)) | frozenset( + (passthrough_model_id,) if passthrough_model_id is not None else () + ) + + +async def _listed_model_mapping_keys( + llm_router: Router | None, + user_api_key_dict: UserAPIKeyAuth, + custom_llm_provider: str | None, + target_model_names: str | None, + routed_model: str | None, +) -> frozenset[str] | None | _UnscopedFileList: + if routed_model is not None and llm_router is not None: + return _model_mapping_keys_serving_model(llm_router, routed_model) + if target_model_names: + target_model_name: Final = _single_target_model_name(target_model_names) + if isinstance(target_model_name, _UnscopedFileList): + return target_model_name + if llm_router is None: + return _UnscopedFileList( + status_code=500, detail="LLM Router not initialized. Ensure models added to proxy." + ) + await get_authorized_credentials_for_model( + llm_router=llm_router, + model_id=target_model_name, + user_api_key_dict=user_api_key_dict, + operation_context="file list", + ) + return _model_mapping_keys_serving_model(llm_router, target_model_name) + if custom_llm_provider is None: + return None + return _model_mapping_keys_on_provider(llm_router, custom_llm_provider) + + @router.get( "/{provider}/v1/files", dependencies=[Depends(user_api_key_auth)], @@ -1697,6 +1836,8 @@ async def list_files( purpose: str | None = None, limit: int | None = None, after: str | None = None, + after_id: str | None = None, + before_id: str | None = None, ): """ Returns information about a specific file. that can be used across - Assistants API, Batch API @@ -1739,6 +1880,10 @@ async def list_files( ) response: Any | None = None + owned_files_hook: Final = proxy_logging_obj.get_proxy_hook("managed_files") + list_owned_files_only: Final = litellm.require_managed_files is True and ( + isinstance(owned_files_hook, BaseFileEndpoints) or not user_api_key_has_admin_view(user_api_key_dict) + ) # Check for model-based credential routing (no file_id encoding check for list) should_route, model_used, _, credentials = await handle_model_based_routing( @@ -1750,7 +1895,39 @@ async def list_files( check_file_id_encoding=False, ) - if should_route and credentials is not None: + if list_owned_files_only: + if after and after_id and after != after_id: + raise HTTPException( + status_code=400, + detail="Pass the list cursor as either 'after' or 'after_id', not both with different ids.", + ) + if not isinstance(owned_files_hook, BaseFileEndpoints): + raise HTTPException( + status_code=500, + detail="Managed file listing is unavailable, so require_managed_files cannot scope the list to the caller.", + ) + requested_custom_llm_provider: Final = await _requested_custom_llm_provider( + request=request, provider=provider + ) + listed_model_mapping_keys: Final = await _listed_model_mapping_keys( + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=requested_custom_llm_provider, + target_model_names=target_model_names, + routed_model=model_used if should_route else None, + ) + if isinstance(listed_model_mapping_keys, _UnscopedFileList): + _raise_unscoped_file_list(listed_model_mapping_keys) + response = await _list_owned_managed_files( # rebind-ok: each dispatch branch binds response + managed_files_obj=owned_files_hook, + user_api_key_dict=user_api_key_dict, + purpose=purpose, + limit=limit if limit is not None else _default_file_list_limit(requested_custom_llm_provider), + after=after or after_id, + before=before_id, + model_mapping_keys=listed_model_mapping_keys, + ) + elif should_route and credentials is not None: # Use model-based routing with credentials from config prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True) response = await litellm.afile_list( @@ -1762,12 +1939,9 @@ async def list_files( verbose_proxy_logger.debug("Listed files using model: %s", model_used) elif target_model_names and isinstance(target_model_names, str): - target_model_names_list: Final = target_model_names.split(",") - if len(target_model_names_list) != 1: - raise HTTPException( - status_code=400, - detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']", - ) + target_model_name: Final = _single_target_model_name(target_model_names) + if isinstance(target_model_name, _UnscopedFileList): + _raise_unscoped_file_list(target_model_name) if llm_router is None: raise HTTPException( status_code=500, @@ -1775,7 +1949,7 @@ async def list_files( ) credentials = await get_authorized_credentials_for_model( llm_router=llm_router, - model_id=target_model_names_list[0], + model_id=target_model_name, user_api_key_dict=user_api_key_dict, operation_context="file list", ) @@ -1786,20 +1960,17 @@ async def list_files( **data, ) else: - custom_llm_provider: Final = ( - provider - or get_custom_llm_provider_from_request_headers(request=request) - or get_custom_llm_provider_from_request_query(request=request) - or await get_custom_llm_provider_from_request_body(request=request) - ) + custom_llm_provider: Final = await _requested_custom_llm_provider(request=request, provider=provider) managed_files_obj: Final = proxy_logging_obj.get_proxy_hook("managed_files") if custom_llm_provider is None and isinstance(managed_files_obj, BaseFileEndpoints): - response = await managed_files_obj.afile_list( - purpose=purpose, - litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + response = await _list_owned_managed_files( # rebind-ok: each dispatch branch binds response + managed_files_obj=managed_files_obj, user_api_key_dict=user_api_key_dict, + purpose=purpose, limit=limit, after=after, + before=None, + model_mapping_keys=None, ) else: resolved_custom_llm_provider: Final = custom_llm_provider or "openai" diff --git a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py index a5a04fb68d2..9790c150ae1 100644 --- a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py +++ b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py @@ -242,6 +242,12 @@ def _passthrough_sentinel_model_id(provider: str) -> str: return f"_passthrough_{provider}" +def passthrough_file_model_id(provider: str) -> str | None: + if (provider, "/v1/files") not in _LIST_ROUTE_TABLE: + return None + return _passthrough_sentinel_model_id(provider) + + # Key under which the provider marker is stored in a file row's model_mappings. # Its value lands in flat_model_file_ids (built from model_mappings.values()), # giving the file table a DB-queryable provider scope it otherwise lacks. diff --git a/tests/integration/security/test_managed_file_listing.py b/tests/integration/security/test_managed_file_listing.py new file mode 100644 index 00000000000..88c5126dae0 --- /dev/null +++ b/tests/integration/security/test_managed_file_listing.py @@ -0,0 +1,265 @@ +import json +import uuid +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from itertools import count, islice +from pathlib import Path +from types import MappingProxyType +from typing import Final +from urllib.parse import parse_qs, urlsplit + +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from anthropic import Anthropic +from openai import OpenAI +from pydantic import BaseModel, ConfigDict, JsonValue + +pytestmark: Final = pytest.mark.shares_worker_state + +UNMANAGED_FILE: Final = "file-uploaded-straight-to-the-provider" +UPLOAD: Final = b"isolation check\n" + + +@dataclass(frozen=True, slots=True) +class Deployment: + provider: str + model: str + litellm_model: str + api_key: str + api_path: str + + +OPENAI: Final = Deployment("openai", "gpt-4.1-managed-listing", "openai/gpt-4.1", "sk-managed-listing", "/v1") +ANTHROPIC: Final = Deployment( + "anthropic", "claude-managed-listing", "anthropic/claude-haiku-5-5", "sk-ant-managed-listing", "" +) +DEPLOYMENTS: Final = (pytest.param(OPENAI, id="openai"), pytest.param(ANTHROPIC, id="anthropic")) + + +@dataclass(frozen=True, slots=True) +class ListCall: + path: str + params: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + headers: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + + +LIST_FORMS: Final = ( + pytest.param(lambda d: ListCall(f"/{d.provider}/v1/files"), id="provider-path"), + pytest.param(lambda d: ListCall("/v1/files", params={"provider": d.provider}), id="provider-query"), + pytest.param(lambda d: ListCall("/v1/files", headers={"custom-llm-provider": d.provider}), id="provider-header"), + pytest.param(lambda d: ListCall("/v1/files", params={"target_model_names": d.model}), id="target-model-names"), + pytest.param(lambda d: ListCall("/v1/files", headers={"x-litellm-model": d.model}), id="model-header"), +) + + +class FilePage(BaseModel): + model_config = ConfigDict(extra="ignore", frozen=True) + + data: list[dict[str, JsonValue]] + has_more: bool + + @property + def ids(self) -> list[str]: + return [str(entry["id"]) for entry in self.data] + + +def _openai_file(file_id: str) -> dict[str, JsonValue]: + return { + "id": file_id, + "object": "file", + "bytes": len(UPLOAD), + "created_at": 1700000000, + "filename": "notes.txt", + "purpose": "user_data", + "status": "processed", + } + + +def _anthropic_file(file_id: str) -> dict[str, JsonValue]: + return { + "id": file_id, + "type": "file", + "filename": "notes.txt", + "mime_type": "text/plain", + "size_bytes": len(UPLOAD), + "created_at": "2026-01-01T00:00:00Z", + "downloadable": False, + } + + +def _reply(body: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(body).encode()) + + +def _list_reply(provider: str, file_ids: tuple[str, ...]) -> Reply: + if provider == ANTHROPIC.provider: + return _reply( + { + "data": [_anthropic_file(file_id) for file_id in file_ids], + "has_more": False, + "first_id": file_ids[0], + "last_id": file_ids[-1], + } + ) + return _reply({"object": "list", "data": [_openai_file(file_id) for file_id in file_ids]}) + + +def _responder(provider_files: Mapping[str, tuple[str, ...]]) -> Callable[[Request], Reply]: + uploads: Final = {provider: count() for provider in provider_files} + + def respond(request: Request) -> Reply: + target: Final = urlsplit(request.target) + path: Final = target.path + provider: Final = ANTHROPIC.provider if "x-api-key" in request.headers else OPENAI.provider + files: Final = provider_files[provider] + if (request.method, path) == ("GET", "/v1/models"): + return _reply({"object": "list", "data": []}) + if (request.method, path) == ("POST", "/v1/files"): + uploaded: Final = files[next(uploads[provider])] + return _reply(_anthropic_file(uploaded) if provider == ANTHROPIC.provider else _openai_file(uploaded)) + if (request.method, path) == ("GET", "/v1/files"): + limit: Final = int(parse_qs(target.query).get("limit", ["10000"])[0]) + return _list_reply(provider, (UNMANAGED_FILE, *reversed(files))[:limit]) + return Reply(status=404, body=json.dumps({"error": {"message": f"unscripted {request.target}"}}).encode()) + + return respond + + +def _config(wire: Wire) -> dict[str, JsonValue]: + return { + "model_list": [ + { + "model_name": deployment.model, + "litellm_params": { + "model": deployment.litellm_model, + "api_key": deployment.api_key, + "api_base": f"{wire.url}{deployment.api_path}", + }, + } + for deployment in (OPENAI, ANTHROPIC) + ], + "litellm_settings": {"require_managed_files": True}, + "general_settings": {"master_key": "os.environ/LITELLM_MASTER_KEY", "database_url": "os.environ/DATABASE_URL"}, + } + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + owner: str + intruder: str + files_by_provider: Mapping[str, tuple[str, ...]] + every_file: tuple[str, ...] + + @property + def base_url(self) -> str: + return str(self.proxy.client.base_url).rstrip("/") + + def list(self, call: ListCall, key: str | None = None) -> FilePage: + response: Final = self.proxy.request("GET", call.path, key=key, params=call.params, headers=call.headers) + assert response.status_code == 200, response.text + return FilePage.model_validate_json(response.text) + + +def _upload(proxy: Gateway, key: str, deployments: tuple[Deployment, ...]) -> str: + with OpenAI(base_url=f"{str(proxy.client.base_url).rstrip('/')}/openai/v1", api_key=key, max_retries=0) as client: + uploaded: Final = client.files.with_raw_response.create( + file=("notes.txt", UPLOAD, "text/plain"), + purpose="user_data", + extra_body={"target_model_names": ",".join(deployment.model for deployment in deployments)}, + ) + assert uploaded.status_code == 200, uploaded.text + return uploaded.parse().id + + +@contextmanager +def _rig(directory: Path) -> Iterator[Rig]: + build: Final = uuid.uuid4().hex + provider_files: Final = { + OPENAI.provider: (f"file-older-{build}", f"file-newer-{build}"), + ANTHROPIC.provider: (f"file_older_{build}", f"file_newer_{build}"), + } + with gateway_from_environment() as shared, wire_server(_responder(provider_files)) as wire: + config: Final = directory / "config.yaml" + config.write_text(yaml.safe_dump(_config(wire))) + with owned_proxy(shared, directory, {}, config=config) as proxy, proxy.scenario() as scenario: + owner: Final = scenario.key(user_id=scenario.user()) + intruder: Final = scenario.key(user_id=scenario.user()) + on_both: Final = _upload(proxy, owner, (OPENAI, ANTHROPIC)) + on_openai: Final = _upload(proxy, owner, (OPENAI,)) + on_anthropic: Final = _upload(proxy, owner, (ANTHROPIC,)) + yield Rig( + proxy, + owner, + intruder, + MappingProxyType( + {OPENAI.provider: (on_openai, on_both), ANTHROPIC.provider: (on_anthropic, on_both)} + ), + (on_anthropic, on_openai, on_both), + ) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with _rig(tmp_path_factory.mktemp("managed-file-listing")) as built: + yield built + + +@pytest.mark.parametrize("deployment", DEPLOYMENTS) +@pytest.mark.parametrize("form", LIST_FORMS) +def test_another_key_lists_none_of_the_files_on_the_shared_key( + rig: Rig, deployment: Deployment, form: Callable[[Deployment], ListCall] +) -> None: + page: Final = rig.list(form(deployment), key=rig.intruder) + assert (page.ids, page.has_more) == ([], False) + + +@pytest.mark.parametrize("deployment", DEPLOYMENTS) +@pytest.mark.parametrize("form", LIST_FORMS) +def test_owner_lists_their_own_managed_files_on_that_provider_or_model_newest_first( + rig: Rig, deployment: Deployment, form: Callable[[Deployment], ListCall] +) -> None: + page: Final = rig.list(form(deployment), key=rig.owner) + assert (page.ids, page.has_more) == (list(rig.files_by_provider[deployment.provider]), False) + + +def test_owner_lists_every_managed_file_when_no_provider_or_model_is_named(rig: Rig) -> None: + page: Final = rig.list(ListCall("/v1/files"), key=rig.owner) + assert (page.ids, page.has_more) == (list(rig.every_file), False) + + +@pytest.mark.parametrize("cursor_param", ["after", "after_id"]) +def test_owner_pages_through_a_provider_list_with_limit_and_a_cursor(rig: Rig, cursor_param: str) -> None: + first: Final = rig.list(ListCall("/openai/v1/files", params={"limit": "1"}), key=rig.owner) + second: Final = rig.list( + ListCall("/openai/v1/files", params={"limit": "1", cursor_param: first.ids[0]}), key=rig.owner + ) + expected: Final = rig.files_by_provider[OPENAI.provider] + assert [(page.ids, page.has_more) for page in (first, second)] == [ + ([expected[0]], True), + ([expected[1]], False), + ] + + +def test_anthropic_sdk_auto_pages_the_owners_files_to_the_end(rig: Rig) -> None: + with Anthropic(base_url=f"{rig.base_url}/anthropic", api_key=rig.owner, max_retries=0) as client: + listed: Final = [file.id for file in islice(client.beta.files.list(limit=1), 5)] + assert listed == list(rig.files_by_provider[ANTHROPIC.provider]) + + +def test_anthropic_sdk_pages_backwards_to_the_newest_file_and_stops(rig: Rig) -> None: + newest_first: Final = rig.files_by_provider[ANTHROPIC.provider] + with Anthropic(base_url=f"{rig.base_url}/anthropic", api_key=rig.owner, max_retries=0) as client: + listed: Final = [file.id for file in islice(client.beta.files.list(limit=1, before_id=newest_first[-1]), 5)] + assert listed == list(reversed(newest_first[:-1])) + + +@pytest.mark.parametrize("deployment", DEPLOYMENTS) +def test_master_key_lists_managed_files_instead_of_the_provider_key(rig: Rig, deployment: Deployment) -> None: + listed: Final = rig.list(ListCall(f"/{deployment.provider}/v1/files")).ids + assert UNMANAGED_FILE not in listed + assert set(rig.files_by_provider[deployment.provider]) <= set(listed) diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index 01f20689e89..4a35c95efc5 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -649,6 +649,22 @@ async def test_afile_list_denies_a_caller_with_no_identity_at_all(): assert table.find_many_calls == [] +@pytest.mark.asyncio +async def test_afile_list_reads_no_rows_when_no_deployment_can_hold_a_file() -> None: + managed_files, table = _make_managed_files_over_rows([_make_managed_file_row("unified-mine")]) + + response: Final = await managed_files.afile_list( + purpose=None, + litellm_parent_otel_span=None, + user_api_key_dict=_make_user_api_key_dict(), + model_mapping_keys=frozenset(), + ) + + assert response.data == [] + assert response.has_more is False + assert table.find_many_calls == [] + + @pytest.mark.asyncio async def test_afile_list_filters_by_purpose(): managed_files, _ = _make_managed_files_over_rows( diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py index d47db69257b..a70859df3e9 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py @@ -12,6 +12,7 @@ from pytest_mock import MockerFixture import litellm from litellm import Router +from litellm.constants import ANTHROPIC_DEFAULT_FILE_LIST_LIMIT from litellm.files.types import FileContentStreamingResult from litellm.proxy._types import LiteLLM_UserTableFiltered, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.hooks import get_proxy_hook @@ -4969,6 +4970,7 @@ def _managed_file_row( stored_file_id: Optional[str] = None, created_by: str = "test-user", team_id: Optional[str] = None, + model_mappings: Optional[dict[str, str]] = None, ) -> MagicMock: file_object = OpenAIFileObject( id=stored_file_id or unified_file_id, @@ -4984,6 +4986,7 @@ def _managed_file_row( file_object=file_object.model_dump(), created_by=created_by, team_id=team_id, + model_mappings=model_mappings or {}, ) @@ -5010,7 +5013,9 @@ class _ManagedFileTableOverRows: return next(iter(self._owned_rows(where)), None) async def find_many(self, where, take=None, order=None, cursor=None, skip=0): - rows = self._owned_rows(where) + owned_rows = self._owned_rows(where) + oldest_first = bool(order) and order[0].get("created_at") == "asc" + rows = owned_rows[::-1] if oldest_first else owned_rows if cursor is not None: start = next( index @@ -6263,3 +6268,369 @@ def test_create_file_passthrough_fails_closed_when_guardrails_would_scan_the_bat assert error["param"] == "passthrough" assert "guardrails" in error["message"] assert forwarded_calls == [] + + +def _callers_files_on_every_deployment() -> list[MagicMock]: + return [ + _managed_file_row("unified-gpt-newer", model_mappings={"gpt-3.5-turbo-id": "file-gpt-newer"}), + _managed_file_row("unified-azure", model_mappings={"azure-gpt-3-5-turbo-id": "file-azure"}), + _managed_file_row("unified-gemini", model_mappings={"gemini-2.0-flash-id": "file-gemini"}), + _managed_file_row( + "unified-theirs", created_by="other-user", model_mappings={"gpt-3.5-turbo-id": "file-theirs"} + ), + _managed_file_row("unified-gpt-older", model_mappings={"gpt-3.5-turbo-id": "file-gpt-older"}), + ] + + +def _list_files_as_test_user(path: str, params: dict[str, str], headers: dict[str, str]) -> httpx.Response: + import litellm.proxy.proxy_server as ps + + try: + return client.get(path, params=params, headers={"Authorization": "Bearer test-key", **headers}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.parametrize( + ("path", "params", "headers", "expected_ids"), + [ + ("/openai/v1/files", {}, {}, ["unified-gpt-newer", "unified-gpt-older"]), + ("/v1/files", {"provider": "azure"}, {}, ["unified-azure"]), + ("/v1/files", {}, {"custom-llm-provider": "gemini"}, ["unified-gemini"]), + ("/v1/files", {"target_model_names": "gemini-2.0-flash"}, {}, ["unified-gemini"]), + ("/v1/files", {}, {"x-litellm-model": "gpt-3.5-turbo"}, ["unified-gpt-newer", "unified-gpt-older"]), + ("/v1/files", {}, {}, ["unified-gpt-newer", "unified-azure", "unified-gemini", "unified-gpt-older"]), + ], + ids=["provider-path", "provider-query", "provider-header", "target-model-names", "model-header", "no-provider"], +) +def test_require_managed_files_lists_the_callers_files_on_the_named_provider_or_model( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + llm_router: Router, + path: str, + params: dict[str, str], + headers: dict[str, str], + expected_ids: list[str], +) -> None: + _, provider_list = _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, _callers_files_on_every_deployment() + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + response: Final = _list_files_as_test_user(path, params, headers) + + assert response.status_code == 200, response.text + assert [file["id"] for file in response.json()["data"]] == expected_ids + provider_list.assert_not_awaited() + + +def _router_with_wildcard_and_unprefixed_anthropic_deployments() -> Router: + unprefixed_anthropic_model: Final = next(model for model in sorted(litellm.anthropic_models) if "/" not in model) + return Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": {"model": "openai/gpt-3.5-turbo", "api_key": "openai_api_key"}, + "model_info": {"id": "gpt-3.5-turbo-id"}, + }, + { + "model_name": "anthropic/*", + "litellm_params": {"model": "anthropic/*", "api_key": "anthropic_api_key"}, + "model_info": {"id": "anthropic-wildcard-id"}, + }, + { + "model_name": "claude-unprefixed", + "litellm_params": {"model": unprefixed_anthropic_model, "api_key": "anthropic_api_key"}, + "model_info": {"id": "claude-unprefixed-id"}, + }, + ] + ) + + +def _callers_files_on_wildcard_unprefixed_and_passthrough_deployments() -> list[MagicMock]: + return [ + _managed_file_row("unified-wildcard", model_mappings={"anthropic-wildcard-id": "file-wildcard"}), + _managed_file_row("unified-unprefixed", model_mappings={"claude-unprefixed-id": "file-unprefixed"}), + _managed_file_row("unified-gpt", model_mappings={"gpt-3.5-turbo-id": "file-gpt"}), + _managed_file_row( + "unified-passthrough", + model_mappings={ + "_passthrough_openai": "file-passthrough", + "_passthrough_provider_marker": "_passthrough_provider:openai", + }, + ), + _managed_file_row("unified-storage", model_mappings={"claude-unprefixed": "litellm_db://unified-storage"}), + ] + + +@pytest.mark.parametrize( + ("path", "params", "headers", "expected_ids"), + [ + ("/v1/files", {"target_model_names": "anthropic/claude-haiku-5-5"}, {}, ["unified-wildcard"]), + ("/v1/files", {}, {"x-litellm-model": "anthropic/claude-haiku-5-5"}, ["unified-wildcard"]), + ("/anthropic/v1/files", {}, {}, ["unified-wildcard", "unified-unprefixed", "unified-storage"]), + ("/openai/v1/files", {}, {}, ["unified-gpt", "unified-passthrough"]), + ("/v1/files", {"target_model_names": "claude-unprefixed"}, {}, ["unified-unprefixed", "unified-storage"]), + ], + ids=[ + "wildcard-target-model-names", + "wildcard-model-header", + "unprefixed-provider", + "passthrough-provider", + "storage-backed-model", + ], +) +def test_require_managed_files_lists_the_callers_files_wherever_the_router_put_them( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + path: str, + params: dict[str, str], + headers: dict[str, str], + expected_ids: list[str], +) -> None: + _, provider_list = _setup_unscoped_list_files_route_over_real_hook( + mocker, + monkeypatch, + _router_with_wildcard_and_unprefixed_anthropic_deployments(), + _callers_files_on_wildcard_unprefixed_and_passthrough_deployments(), + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + response: Final = _list_files_as_test_user(path, params, headers) + + assert response.status_code == 200, response.text + assert [file["id"] for file in response.json()["data"]] == expected_ids + provider_list.assert_not_awaited() + + +@pytest.mark.parametrize( + ("path", "params"), + [ + ("/antropic/v1/files", {}), + ("/openai/v1/files", {"after": "unified-gpt", "after_id": "unified-wildcard"}), + ("/v1/files", {"target_model_names": "gpt-3.5-turbo,claude-unprefixed"}), + ], + ids=["unknown-provider", "conflicting-after-cursors", "two-target-model-names"], +) +def test_require_managed_files_rejects_a_list_it_cannot_scope( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, path: str, params: dict[str, str] +) -> None: + _, provider_list = _setup_unscoped_list_files_route_over_real_hook( + mocker, + monkeypatch, + _router_with_wildcard_and_unprefixed_anthropic_deployments(), + _callers_files_on_wildcard_unprefixed_and_passthrough_deployments(), + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + response: Final = _list_files_as_test_user(path, params, {}) + + assert response.status_code == 400, response.text + provider_list.assert_not_awaited() + + +@pytest.mark.parametrize("cursor_param", ["after", "after_id"]) +def test_require_managed_files_pages_a_provider_list_with_either_cursor_param( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, llm_router: Router, cursor_param: str +) -> None: + import litellm.proxy.proxy_server as ps + + _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, _callers_files_on_every_deployment() + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + try: + first_page: Final = client.get( + "/openai/v1/files", params={"limit": "1"}, headers={"Authorization": "Bearer test-key"} + ) + second_page: Final = client.get( + "/openai/v1/files", + params={"limit": "1", cursor_param: first_page.json()["last_id"]}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert (first_page.status_code, second_page.status_code) == (200, 200), (first_page.text, second_page.text) + assert [file["id"] for file in first_page.json()["data"]] == ["unified-gpt-newer"] + assert first_page.json()["has_more"] is True + assert [file["id"] for file in second_page.json()["data"]] == ["unified-gpt-older"] + assert second_page.json()["has_more"] is False + + +def test_require_managed_files_pages_toward_newer_files_with_before_id( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, llm_router: Router +) -> None: + import litellm.proxy.proxy_server as ps + + _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, _callers_files_on_every_deployment() + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + try: + page_before_oldest: Final = client.get( + "/v1/files", + params={"limit": "2", "before_id": "unified-gpt-older"}, + headers={"Authorization": "Bearer test-key"}, + ) + newest_page: Final = client.get( + "/v1/files", + params={"limit": "2", "before_id": page_before_oldest.json()["first_id"]}, + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert (page_before_oldest.status_code, newest_page.status_code) == (200, 200), ( + page_before_oldest.text, + newest_page.text, + ) + assert [file["id"] for file in page_before_oldest.json()["data"]] == ["unified-azure", "unified-gemini"] + assert page_before_oldest.json()["has_more"] is True + assert [file["id"] for file in newest_page.json()["data"]] == ["unified-gpt-newer"] + assert newest_page.json()["has_more"] is False + + +def test_managed_file_list_rejects_both_cursor_directions( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, llm_router: Router +) -> None: + _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, _callers_files_on_every_deployment() + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + response: Final = _list_files_as_test_user( + "/v1/files", {"after": "unified-gpt-newer", "before_id": "unified-gpt-older"}, {} + ) + + assert response.status_code == 400, response.text + assert "not both" in response.json()["error"]["message"] + + +@pytest.mark.parametrize( + ("params", "headers"), + [({"target_model_names": "gemini-2.0-flash"}, {}), ({}, {"x-litellm-model": "gemini-2.0-flash"})], + ids=["target-model-names", "model-header"], +) +def test_require_managed_files_list_rejects_a_model_the_key_cannot_use( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + llm_router: Router, + params: dict[str, str], + headers: dict[str, str], +) -> None: + import litellm.proxy.proxy_server as ps + + _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, _callers_files_on_every_deployment() + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + models=["gpt-3.5-turbo"], + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + response: Final = _list_files_as_test_user("/v1/files", params, headers) + + assert response.status_code == 403, response.text + assert "unified-gemini" not in response.text + + +def test_require_managed_files_list_fails_closed_without_the_managed_files_hook( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, llm_router: Router +) -> None: + proxy_logging_obj, provider_list = _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, _callers_files_on_every_deployment() + ) + proxy_logging_obj.proxy_hook_mapping.pop("managed_files") + provider_list.return_value = [_managed_file("file-on-the-shared-key")] + monkeypatch.setattr("litellm.require_managed_files", True) + + response: Final = _list_files_as_test_user("/openai/v1/files", {}, {}) + + assert response.status_code == 500, response.text + assert "file-on-the-shared-key" not in response.text + assert "require_managed_files cannot scope the list to the caller" in response.json()["error"]["message"] + provider_list.assert_not_awaited() + + +@pytest.mark.parametrize( + ("hook_loaded", "expected_ids"), + [ + (True, ["unified-gpt-newer", "unified-theirs", "unified-gpt-older"]), + (False, ["file-on-the-shared-key"]), + ], +) +def test_require_managed_files_lists_managed_files_for_an_admin_and_the_provider_list_without_the_hook( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + llm_router: Router, + hook_loaded: bool, + expected_ids: list[str], +) -> None: + import litellm.proxy.proxy_server as ps + + proxy_logging_obj, provider_list = _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, _callers_files_on_every_deployment() + ) + if not hook_loaded: + proxy_logging_obj.proxy_hook_mapping.pop("managed_files") + provider_list.return_value = [_managed_file("file-on-the-shared-key")] + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-user" + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + response: Final = _list_files_as_test_user("/openai/v1/files", {}, {}) + + assert response.status_code == 200, response.text + assert [file["id"] for file in response.json()["data"]] == expected_ids + + +def test_managed_file_list_without_require_managed_files_ignores_cursor_params_openai_does_not_define( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, llm_router: Router +) -> None: + _setup_unscoped_list_files_route_over_real_hook( + mocker, monkeypatch, llm_router, _callers_files_on_every_deployment() + ) + monkeypatch.setattr("litellm.require_managed_files", False) + + response: Final = _list_files_as_test_user( + "/v1/files", {"after": "unified-gpt-newer", "after_id": "unified-gemini", "before_id": "file-raw"}, {} + ) + + assert response.status_code == 200, response.text + assert [file["id"] for file in response.json()["data"]] == ["unified-azure", "unified-gemini", "unified-gpt-older"] + + +def test_require_managed_files_pages_the_anthropic_list_at_the_anthropic_default_size( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm.proxy.proxy_server as ps + + _setup_unscoped_list_files_route_over_real_hook( + mocker, + monkeypatch, + _router_with_wildcard_and_unprefixed_anthropic_deployments(), + [ + _managed_file_row(f"unified-claude-{index:03d}", model_mappings={"claude-unprefixed-id": f"file-{index:03d}"}) + for index in range(ANTHROPIC_DEFAULT_FILE_LIST_LIMIT + 1) + ], + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + try: + anthropic_page: Final = client.get("/anthropic/v1/files", headers={"Authorization": "Bearer test-key"}) + unified_list: Final = client.get("/v1/files", headers={"Authorization": "Bearer test-key"}) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert (anthropic_page.status_code, unified_list.status_code) == (200, 200), (anthropic_page.text, unified_list.text) + assert len(anthropic_page.json()["data"]) == ANTHROPIC_DEFAULT_FILE_LIST_LIMIT + assert anthropic_page.json()["has_more"] is True + assert len(unified_list.json()["data"]) == ANTHROPIC_DEFAULT_FILE_LIST_LIMIT + 1 + assert unified_list.json()["has_more"] is False diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 13a8a8b993d..8a1d10e9293 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -59815,6 +59815,8 @@ export interface operations { purpose?: string | null; limit?: number | null; after?: string | null; + after_id?: string | null; + before_id?: string | null; }; header?: never; path?: never; @@ -79007,6 +79009,8 @@ export interface operations { purpose?: string | null; limit?: number | null; after?: string | null; + after_id?: string | null; + before_id?: string | null; }; header?: never; path?: never; @@ -86341,6 +86345,8 @@ export interface operations { purpose?: string | null; limit?: number | null; after?: string | null; + after_id?: string | null; + before_id?: string | null; }; header?: never; path: {