From ec1c55301ecf5faec7526c93ac3feec0436275a5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 02:35:18 -0700 Subject: [PATCH] fix(files): filter provider and model lists in the query A non-admin list scoped to a provider or model now asks Postgres only for the caller's rows that map to one of that scope's deployments or models, or that live in file storage, instead of reading every owned row and filtering them in Python. A caller with many files on other providers no longer waits for the whole set to be read. Team members' files are matched in the same query. --- .../proxy/hooks/managed_files.py | 112 +++++++++++++++--- .../test_files_endpoint.py | 66 ++++++++++- 2 files changed, 159 insertions(+), 19 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 0716666284b..04ae7e77b05 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -27,7 +27,7 @@ import httpx from fastapi import HTTPException from openai import APIConnectionError, AsyncOpenAI from openai.types.file_deleted import FileDeleted -from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, Unpack import litellm @@ -359,6 +359,8 @@ class _ManagedResourceDatabase(Protocol): @property def litellm_managedfiletable(self) -> _ManagedFileTableActions: ... + async def query_raw(self, query: str, *args: object) -> object: ... + @property def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ... @@ -386,6 +388,75 @@ def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _Managed return prisma_client.db.litellm_managedfiletable +class _ListedFileRow(Protocol): + @property + def unified_file_id(self) -> str: ... + + @property + def file_object(self) -> object: ... + + @property + def model_mappings(self) -> Mapping[str, object]: ... + + @property + def storage_backend(self) -> str | None: ... + + +class _ScopedFileRow(BaseModel): + model_config = ConfigDict(frozen=True) + + unified_file_id: str + file_object: JsonValue = None + model_mappings: Mapping[str, JsonValue] + storage_backend: str | None = None + + +_SCOPED_FILE_ROWS: Final = TypeAdapter(tuple[_ScopedFileRow, ...]) + + +def _owner_condition(owner_filter: Mapping[str, object]) -> tuple[str, tuple[str, ...]]: + match owner_filter: + case {"OR": [{"created_by": str() as user_id}, {"team_id": str() as team_id}]}: + return '("created_by" = $1 OR "team_id" = $2)', (user_id, team_id) + case {"created_by": str() as user_id}: + return '"created_by" = $1', (user_id,) + case {"team_id": str() as team_id}: + return '"team_id" = $1', (team_id,) + case _: + return ("FALSE", ()) if owner_filter else ("TRUE", ()) + + +async def _scoped_file_rows( + prisma_client: _ManagedResourcePrismaClient, + owner_filter: Mapping[str, object], + model_mapping_keys: frozenset[str], + cursor_id: str | None, + direction: Literal["asc", "desc"], + take: int, +) -> tuple[_ScopedFileRow, ...]: + owner_condition, owner_params = _owner_condition(owner_filter) + keys_param: Final = len(owner_params) + 1 + comparison: Final = "<" if direction == "desc" else ">" + cursor_condition: Final = ( + f'AND ("created_at", "unified_file_id") {comparison} (SELECT "created_at", "unified_file_id" ' + f'FROM "LiteLLM_ManagedFileTable" WHERE "unified_file_id" = ${keys_param + 1}) ' + if cursor_id + else "" + ) + order: Final = direction.upper() + rows: Final = await prisma_client.db.query_raw( + 'SELECT "unified_file_id", "file_object", "model_mappings", "storage_backend" FROM "LiteLLM_ManagedFileTable" ' + f'WHERE {owner_condition} AND ("model_mappings" ?| ${keys_param}::text[] OR "storage_backend" IS NOT NULL) ' + f'{cursor_condition}ORDER BY "created_at" {order}, "unified_file_id" {order} ' + f"LIMIT ${keys_param + (2 if cursor_id else 1)}", + *owner_params, + sorted(model_mapping_keys), + *((cursor_id,) if cursor_id else ()), + take, + ) + return _SCOPED_FILE_ROWS.validate_python(rows) + + def _iter_provider_file_id_pairs( rows: Sequence[_ManagedFileRow], requested_provider_file_ids: frozenset[str], @@ -2262,12 +2333,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): every file on the proxy still reads one bounded page at a time. ``after`` pages toward older files and ``before`` toward newer ones, each page still ordered newest first. - ``purpose`` is applied after parsing, because the managed file table - keeps it inside the ``file_object`` blob instead of a column, and - ``model_scope`` keeps only files whose ``model_mappings``, a JSON - column, name one of its deployment ids or model names, or for a - storage-backed file a model name the router sends to the listed - provider. Rows + ``model_scope`` is applied in the query, so only rows whose + ``model_mappings`` name one of its deployment ids or model names, and + storage-backed rows, leave the database; a storage-backed row then + stays only when its model name is one the router sends to the listed + provider. ``purpose`` is applied after parsing, because the managed + file table keeps it inside the ``file_object`` blob instead of a + column. Rows whose blob will not parse drop out there too, so a chunk of rows can yield fewer matches than the page holds. Successive chunks are read until the page is full or the caller's rows run out, which keeps @@ -2276,9 +2348,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): scan has to continue past it, the chunk widens to ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per that many rows instead of one per page. That bound is per query, not - per request: the work is still linear in the rows the caller owns, and - a filter matching nothing reads every one of them, each chunk through - the owner and creation-time indexes instead of a table scan. + per request: a ``purpose`` filter matching nothing still reads every + row that passes the query. """ validate_file_list_limit(limit) validate_file_list_purpose(purpose) @@ -2317,22 +2388,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): matches: Final[List[OpenAIFileObject]] = [] cursor_id = cursor chunk_size = page_size + 1 - direction: Final = "asc" if before else "desc" + direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc" while len(matches) <= page_size: cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {} - chunk = await _managed_file_table(self.prisma_client).find_many( - where=owner_filter, - take=chunk_size, - order=[{"created_at": direction}, {"unified_file_id": direction}], - **cursor_args, + chunk: Sequence[_ListedFileRow] = ( + await _scoped_file_rows( + self.prisma_client, owner_filter, model_scope.model_mapping_keys, cursor_id, direction, chunk_size + ) + if model_scope is not None + else await _managed_file_table(self.prisma_client).find_many( + where=owner_filter, + take=chunk_size, + order=[{"created_at": direction}, {"unified_file_id": direction}], + **cursor_args, + ) ) matches.extend( _public_file_object(parsed_file_object, row.unified_file_id) for row in chunk if ( - model_scope is None - or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None) + model_scope is None or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None) ) and (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None and (purpose is None or parsed_file_object.purpose == purpose) diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py index 27e27ce99eb..4937c1da826 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py @@ -4985,6 +4985,37 @@ class _ManagedFileTableOverRows: rows: Final = ordered_rows[start:] return rows if take is None else rows[:take] + async def query_raw(self, query, *params): + keys_index: Final = next(index for index, param in enumerate(params) if isinstance(param, list)) + owner_ids: Final = frozenset(params[:keys_index]) + model_mapping_keys: Final = frozenset(params[keys_index]) + cursor_id: Final = params[keys_index + 1] if len(params) == keys_index + 3 else None + ordered_rows: Final = [ + row + for row in (self.rows[::-1] if '"created_at" ASC' in query else self.rows) + if not owner_ids or {row.created_by, row.team_id} & owner_ids + ] + start: Final = ( + 0 + if cursor_id is None + else 1 + next(index for index, row in enumerate(ordered_rows) if row.unified_file_id == cursor_id) + ) + listed_rows: Final = [ + row + for row in ordered_rows[start:] + if not model_mapping_keys.isdisjoint(row.model_mappings) + or ('"storage_backend" IS NOT NULL' in query and row.storage_backend is not None) + ] + return [ + { + "unified_file_id": row.unified_file_id, + "file_object": row.file_object, + "model_mappings": row.model_mappings, + "storage_backend": row.storage_backend, + } + for row in listed_rows[: params[-1]] + ] + def _setup_unscoped_list_files_route_over_real_hook( mocker, monkeypatch, llm_router: Router, rows @@ -5006,7 +5037,9 @@ def _setup_unscoped_list_files_route_over_real_hook( managed_files = PROXY_LiteLLMManagedFiles( internal_usage_cache=MagicMock(), prisma_client=MagicMock() ) - managed_files.prisma_client.db.litellm_managedfiletable = _ManagedFileTableOverRows(rows) + managed_file_table = _ManagedFileTableOverRows(rows) + managed_files.prisma_client.db.litellm_managedfiletable = managed_file_table + managed_files.prisma_client.db.query_raw = managed_file_table.query_raw proxy_logging_obj = setup_proxy_logging_object(monkeypatch, llm_router) proxy_logging_obj.proxy_hook_mapping["managed_files"] = managed_files @@ -6284,6 +6317,37 @@ def test_require_managed_files_lists_the_callers_files_on_the_named_provider_or_ provider_list.assert_not_awaited() +def test_require_managed_files_lists_a_teammates_files_on_the_provider_and_no_other_teams( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, llm_router: Router +) -> None: + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + _setup_unscoped_list_files_route_over_real_hook( + mocker, + monkeypatch, + llm_router, + [ + _managed_file_row( + "unified-teammate", created_by="teammate", team_id="team-a", model_mappings={"gpt-3.5-turbo-id": "f-1"} + ), + _managed_file_row("unified-own", model_mappings={"gpt-3.5-turbo-id": "f-2"}), + _managed_file_row( + "unified-other-team", created_by="outsider", team_id="team-b", model_mappings={"gpt-3.5-turbo-id": "f-3"} + ), + ], + ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", user_role=LitellmUserRoles.INTERNAL_USER, user_id="test-user", team_id="team-a" + ) + monkeypatch.setattr("litellm.require_managed_files", True) + + response: Final = _list_files_as_test_user("/openai/v1/files", {}, {}) + + assert response.status_code == 200, response.text + assert [file["id"] for file in response.json()["data"]] == ["unified-teammate", "unified-own"] + + def _router_with_wildcard_and_unprefixed_anthropic_deployments() -> Router: unprefixed_anthropic_model: Final = next(model for model in sorted(litellm.anthropic_models) if "/" not in model) return Router(