fix(files): filter provider and model lists in the query

A non-admin list scoped to a provider or model now asks Postgres only for the
caller's rows that map to one of that scope's deployments or models, or that
live in file storage, instead of reading every owned row and filtering them in
Python. A caller with many files on other providers no longer waits for the
whole set to be read. Team members' files are matched in the same query.
This commit is contained in:
mateo-berri 2026-10-21 02:35:18 -07:00
parent cbe5b9e77d
commit ec1c55301e
2 changed files with 159 additions and 19 deletions

View file

@ -27,7 +27,7 @@ import httpx
from fastapi import HTTPException
from openai import APIConnectionError, AsyncOpenAI
from openai.types.file_deleted import FileDeleted
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, Unpack
import litellm
@ -359,6 +359,8 @@ class _ManagedResourceDatabase(Protocol):
@property
def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
async def query_raw(self, query: str, *args: object) -> object: ...
@property
def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
@ -386,6 +388,75 @@ def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _Managed
return prisma_client.db.litellm_managedfiletable
class _ListedFileRow(Protocol):
@property
def unified_file_id(self) -> str: ...
@property
def file_object(self) -> object: ...
@property
def model_mappings(self) -> Mapping[str, object]: ...
@property
def storage_backend(self) -> str | None: ...
class _ScopedFileRow(BaseModel):
model_config = ConfigDict(frozen=True)
unified_file_id: str
file_object: JsonValue = None
model_mappings: Mapping[str, JsonValue]
storage_backend: str | None = None
_SCOPED_FILE_ROWS: Final = TypeAdapter(tuple[_ScopedFileRow, ...])
def _owner_condition(owner_filter: Mapping[str, object]) -> tuple[str, tuple[str, ...]]:
match owner_filter:
case {"OR": [{"created_by": str() as user_id}, {"team_id": str() as team_id}]}:
return '("created_by" = $1 OR "team_id" = $2)', (user_id, team_id)
case {"created_by": str() as user_id}:
return '"created_by" = $1', (user_id,)
case {"team_id": str() as team_id}:
return '"team_id" = $1', (team_id,)
case _:
return ("FALSE", ()) if owner_filter else ("TRUE", ())
async def _scoped_file_rows(
prisma_client: _ManagedResourcePrismaClient,
owner_filter: Mapping[str, object],
model_mapping_keys: frozenset[str],
cursor_id: str | None,
direction: Literal["asc", "desc"],
take: int,
) -> tuple[_ScopedFileRow, ...]:
owner_condition, owner_params = _owner_condition(owner_filter)
keys_param: Final = len(owner_params) + 1
comparison: Final = "<" if direction == "desc" else ">"
cursor_condition: Final = (
f'AND ("created_at", "unified_file_id") {comparison} (SELECT "created_at", "unified_file_id" '
f'FROM "LiteLLM_ManagedFileTable" WHERE "unified_file_id" = ${keys_param + 1}) '
if cursor_id
else ""
)
order: Final = direction.upper()
rows: Final = await prisma_client.db.query_raw(
'SELECT "unified_file_id", "file_object", "model_mappings", "storage_backend" FROM "LiteLLM_ManagedFileTable" '
f'WHERE {owner_condition} AND ("model_mappings" ?| ${keys_param}::text[] OR "storage_backend" IS NOT NULL) '
f'{cursor_condition}ORDER BY "created_at" {order}, "unified_file_id" {order} '
f"LIMIT ${keys_param + (2 if cursor_id else 1)}",
*owner_params,
sorted(model_mapping_keys),
*((cursor_id,) if cursor_id else ()),
take,
)
return _SCOPED_FILE_ROWS.validate_python(rows)
def _iter_provider_file_id_pairs(
rows: Sequence[_ManagedFileRow],
requested_provider_file_ids: frozenset[str],
@ -2262,12 +2333,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
every file on the proxy still reads one bounded page at a time.
``after`` pages toward older files and ``before`` toward newer ones,
each page still ordered newest first.
``purpose`` is applied after parsing, because the managed file table
keeps it inside the ``file_object`` blob instead of a column, and
``model_scope`` keeps only files whose ``model_mappings``, a JSON
column, name one of its deployment ids or model names, or for a
storage-backed file a model name the router sends to the listed
provider. Rows
``model_scope`` is applied in the query, so only rows whose
``model_mappings`` name one of its deployment ids or model names, and
storage-backed rows, leave the database; a storage-backed row then
stays only when its model name is one the router sends to the listed
provider. ``purpose`` is applied after parsing, because the managed
file table keeps it inside the ``file_object`` blob instead of a
column. Rows
whose blob will not parse drop out there too, so a chunk of rows can
yield fewer matches than the page holds. Successive chunks are read
until the page is full or the caller's rows run out, which keeps
@ -2276,9 +2348,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
scan has to continue past it, the chunk widens to
``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
that many rows instead of one per page. That bound is per query, not
per request: the work is still linear in the rows the caller owns, and
a filter matching nothing reads every one of them, each chunk through
the owner and creation-time indexes instead of a table scan.
per request: a ``purpose`` filter matching nothing still reads every
row that passes the query.
"""
validate_file_list_limit(limit)
validate_file_list_purpose(purpose)
@ -2317,22 +2388,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
matches: Final[List[OpenAIFileObject]] = []
cursor_id = cursor
chunk_size = page_size + 1
direction: Final = "asc" if before else "desc"
direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc"
while len(matches) <= page_size:
cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
chunk = await _managed_file_table(self.prisma_client).find_many(
where=owner_filter,
take=chunk_size,
order=[{"created_at": direction}, {"unified_file_id": direction}],
**cursor_args,
chunk: Sequence[_ListedFileRow] = (
await _scoped_file_rows(
self.prisma_client, owner_filter, model_scope.model_mapping_keys, cursor_id, direction, chunk_size
)
if model_scope is not None
else await _managed_file_table(self.prisma_client).find_many(
where=owner_filter,
take=chunk_size,
order=[{"created_at": direction}, {"unified_file_id": direction}],
**cursor_args,
)
)
matches.extend(
_public_file_object(parsed_file_object, row.unified_file_id)
for row in chunk
if (
model_scope is None
or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
model_scope is None or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
)
and (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
and (purpose is None or parsed_file_object.purpose == purpose)

View file

@ -4985,6 +4985,37 @@ class _ManagedFileTableOverRows:
rows: Final = ordered_rows[start:]
return rows if take is None else rows[:take]
async def query_raw(self, query, *params):
keys_index: Final = next(index for index, param in enumerate(params) if isinstance(param, list))
owner_ids: Final = frozenset(params[:keys_index])
model_mapping_keys: Final = frozenset(params[keys_index])
cursor_id: Final = params[keys_index + 1] if len(params) == keys_index + 3 else None
ordered_rows: Final = [
row
for row in (self.rows[::-1] if '"created_at" ASC' in query else self.rows)
if not owner_ids or {row.created_by, row.team_id} & owner_ids
]
start: Final = (
0
if cursor_id is None
else 1 + next(index for index, row in enumerate(ordered_rows) if row.unified_file_id == cursor_id)
)
listed_rows: Final = [
row
for row in ordered_rows[start:]
if not model_mapping_keys.isdisjoint(row.model_mappings)
or ('"storage_backend" IS NOT NULL' in query and row.storage_backend is not None)
]
return [
{
"unified_file_id": row.unified_file_id,
"file_object": row.file_object,
"model_mappings": row.model_mappings,
"storage_backend": row.storage_backend,
}
for row in listed_rows[: params[-1]]
]
def _setup_unscoped_list_files_route_over_real_hook(
mocker, monkeypatch, llm_router: Router, rows
@ -5006,7 +5037,9 @@ def _setup_unscoped_list_files_route_over_real_hook(
managed_files = PROXY_LiteLLMManagedFiles(
internal_usage_cache=MagicMock(), prisma_client=MagicMock()
)
managed_files.prisma_client.db.litellm_managedfiletable = _ManagedFileTableOverRows(rows)
managed_file_table = _ManagedFileTableOverRows(rows)
managed_files.prisma_client.db.litellm_managedfiletable = managed_file_table
managed_files.prisma_client.db.query_raw = managed_file_table.query_raw
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, llm_router)
proxy_logging_obj.proxy_hook_mapping["managed_files"] = managed_files
@ -6284,6 +6317,37 @@ def test_require_managed_files_lists_the_callers_files_on_the_named_provider_or_
provider_list.assert_not_awaited()
def test_require_managed_files_lists_a_teammates_files_on_the_provider_and_no_other_teams(
mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, llm_router: Router
) -> None:
import litellm.proxy.proxy_server as ps
from litellm.proxy._types import LitellmUserRoles
_setup_unscoped_list_files_route_over_real_hook(
mocker,
monkeypatch,
llm_router,
[
_managed_file_row(
"unified-teammate", created_by="teammate", team_id="team-a", model_mappings={"gpt-3.5-turbo-id": "f-1"}
),
_managed_file_row("unified-own", model_mappings={"gpt-3.5-turbo-id": "f-2"}),
_managed_file_row(
"unified-other-team", created_by="outsider", team_id="team-b", model_mappings={"gpt-3.5-turbo-id": "f-3"}
),
],
)
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
api_key="test-key", user_role=LitellmUserRoles.INTERNAL_USER, user_id="test-user", team_id="team-a"
)
monkeypatch.setattr("litellm.require_managed_files", True)
response: Final = _list_files_as_test_user("/openai/v1/files", {}, {})
assert response.status_code == 200, response.text
assert [file["id"] for file in response.json()["data"]] == ["unified-teammate", "unified-own"]
def _router_with_wildcard_and_unprefixed_anthropic_deployments() -> Router:
unprefixed_anthropic_model: Final = next(model for model in sorted(litellm.anthropic_models) if "/" not in model)
return Router(