mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(files): filter provider and model lists in the query
A non-admin list scoped to a provider or model now asks Postgres only for the caller's rows that map to one of that scope's deployments or models, or that live in file storage, instead of reading every owned row and filtering them in Python. A caller with many files on other providers no longer waits for the whole set to be read. Team members' files are matched in the same query.
This commit is contained in:
parent
cbe5b9e77d
commit
ec1c55301e
2 changed files with 159 additions and 19 deletions
|
|
@ -27,7 +27,7 @@ import httpx
|
|||
from fastapi import HTTPException
|
||||
from openai import APIConnectionError, AsyncOpenAI
|
||||
from openai.types.file_deleted import FileDeleted
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, Unpack
|
||||
|
||||
import litellm
|
||||
|
|
@ -359,6 +359,8 @@ class _ManagedResourceDatabase(Protocol):
|
|||
@property
|
||||
def litellm_managedfiletable(self) -> _ManagedFileTableActions: ...
|
||||
|
||||
async def query_raw(self, query: str, *args: object) -> object: ...
|
||||
|
||||
@property
|
||||
def litellm_managedobjecttable(self) -> _ManagedObjectTableActions: ...
|
||||
|
||||
|
|
@ -386,6 +388,75 @@ def _managed_file_table(prisma_client: _ManagedResourcePrismaClient) -> _Managed
|
|||
return prisma_client.db.litellm_managedfiletable
|
||||
|
||||
|
||||
class _ListedFileRow(Protocol):
|
||||
@property
|
||||
def unified_file_id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def file_object(self) -> object: ...
|
||||
|
||||
@property
|
||||
def model_mappings(self) -> Mapping[str, object]: ...
|
||||
|
||||
@property
|
||||
def storage_backend(self) -> str | None: ...
|
||||
|
||||
|
||||
class _ScopedFileRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
unified_file_id: str
|
||||
file_object: JsonValue = None
|
||||
model_mappings: Mapping[str, JsonValue]
|
||||
storage_backend: str | None = None
|
||||
|
||||
|
||||
_SCOPED_FILE_ROWS: Final = TypeAdapter(tuple[_ScopedFileRow, ...])
|
||||
|
||||
|
||||
def _owner_condition(owner_filter: Mapping[str, object]) -> tuple[str, tuple[str, ...]]:
|
||||
match owner_filter:
|
||||
case {"OR": [{"created_by": str() as user_id}, {"team_id": str() as team_id}]}:
|
||||
return '("created_by" = $1 OR "team_id" = $2)', (user_id, team_id)
|
||||
case {"created_by": str() as user_id}:
|
||||
return '"created_by" = $1', (user_id,)
|
||||
case {"team_id": str() as team_id}:
|
||||
return '"team_id" = $1', (team_id,)
|
||||
case _:
|
||||
return ("FALSE", ()) if owner_filter else ("TRUE", ())
|
||||
|
||||
|
||||
async def _scoped_file_rows(
|
||||
prisma_client: _ManagedResourcePrismaClient,
|
||||
owner_filter: Mapping[str, object],
|
||||
model_mapping_keys: frozenset[str],
|
||||
cursor_id: str | None,
|
||||
direction: Literal["asc", "desc"],
|
||||
take: int,
|
||||
) -> tuple[_ScopedFileRow, ...]:
|
||||
owner_condition, owner_params = _owner_condition(owner_filter)
|
||||
keys_param: Final = len(owner_params) + 1
|
||||
comparison: Final = "<" if direction == "desc" else ">"
|
||||
cursor_condition: Final = (
|
||||
f'AND ("created_at", "unified_file_id") {comparison} (SELECT "created_at", "unified_file_id" '
|
||||
f'FROM "LiteLLM_ManagedFileTable" WHERE "unified_file_id" = ${keys_param + 1}) '
|
||||
if cursor_id
|
||||
else ""
|
||||
)
|
||||
order: Final = direction.upper()
|
||||
rows: Final = await prisma_client.db.query_raw(
|
||||
'SELECT "unified_file_id", "file_object", "model_mappings", "storage_backend" FROM "LiteLLM_ManagedFileTable" '
|
||||
f'WHERE {owner_condition} AND ("model_mappings" ?| ${keys_param}::text[] OR "storage_backend" IS NOT NULL) '
|
||||
f'{cursor_condition}ORDER BY "created_at" {order}, "unified_file_id" {order} '
|
||||
f"LIMIT ${keys_param + (2 if cursor_id else 1)}",
|
||||
*owner_params,
|
||||
sorted(model_mapping_keys),
|
||||
*((cursor_id,) if cursor_id else ()),
|
||||
take,
|
||||
)
|
||||
return _SCOPED_FILE_ROWS.validate_python(rows)
|
||||
|
||||
|
||||
def _iter_provider_file_id_pairs(
|
||||
rows: Sequence[_ManagedFileRow],
|
||||
requested_provider_file_ids: frozenset[str],
|
||||
|
|
@ -2262,12 +2333,13 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
every file on the proxy still reads one bounded page at a time.
|
||||
``after`` pages toward older files and ``before`` toward newer ones,
|
||||
each page still ordered newest first.
|
||||
``purpose`` is applied after parsing, because the managed file table
|
||||
keeps it inside the ``file_object`` blob instead of a column, and
|
||||
``model_scope`` keeps only files whose ``model_mappings``, a JSON
|
||||
column, name one of its deployment ids or model names, or for a
|
||||
storage-backed file a model name the router sends to the listed
|
||||
provider. Rows
|
||||
``model_scope`` is applied in the query, so only rows whose
|
||||
``model_mappings`` name one of its deployment ids or model names, and
|
||||
storage-backed rows, leave the database; a storage-backed row then
|
||||
stays only when its model name is one the router sends to the listed
|
||||
provider. ``purpose`` is applied after parsing, because the managed
|
||||
file table keeps it inside the ``file_object`` blob instead of a
|
||||
column. Rows
|
||||
whose blob will not parse drop out there too, so a chunk of rows can
|
||||
yield fewer matches than the page holds. Successive chunks are read
|
||||
until the page is full or the caller's rows run out, which keeps
|
||||
|
|
@ -2276,9 +2348,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
scan has to continue past it, the chunk widens to
|
||||
``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
|
||||
that many rows instead of one per page. That bound is per query, not
|
||||
per request: the work is still linear in the rows the caller owns, and
|
||||
a filter matching nothing reads every one of them, each chunk through
|
||||
the owner and creation-time indexes instead of a table scan.
|
||||
per request: a ``purpose`` filter matching nothing still reads every
|
||||
row that passes the query.
|
||||
"""
|
||||
validate_file_list_limit(limit)
|
||||
validate_file_list_purpose(purpose)
|
||||
|
|
@ -2317,22 +2388,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
matches: Final[List[OpenAIFileObject]] = []
|
||||
cursor_id = cursor
|
||||
chunk_size = page_size + 1
|
||||
direction: Final = "asc" if before else "desc"
|
||||
direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc"
|
||||
|
||||
while len(matches) <= page_size:
|
||||
cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
|
||||
chunk = await _managed_file_table(self.prisma_client).find_many(
|
||||
where=owner_filter,
|
||||
take=chunk_size,
|
||||
order=[{"created_at": direction}, {"unified_file_id": direction}],
|
||||
**cursor_args,
|
||||
chunk: Sequence[_ListedFileRow] = (
|
||||
await _scoped_file_rows(
|
||||
self.prisma_client, owner_filter, model_scope.model_mapping_keys, cursor_id, direction, chunk_size
|
||||
)
|
||||
if model_scope is not None
|
||||
else await _managed_file_table(self.prisma_client).find_many(
|
||||
where=owner_filter,
|
||||
take=chunk_size,
|
||||
order=[{"created_at": direction}, {"unified_file_id": direction}],
|
||||
**cursor_args,
|
||||
)
|
||||
)
|
||||
matches.extend(
|
||||
_public_file_object(parsed_file_object, row.unified_file_id)
|
||||
for row in chunk
|
||||
if (
|
||||
model_scope is None
|
||||
or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
|
||||
model_scope is None or model_scope.holds(row.model_mappings, stored=row.storage_backend is not None)
|
||||
)
|
||||
and (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
|
||||
and (purpose is None or parsed_file_object.purpose == purpose)
|
||||
|
|
|
|||
|
|
@ -4985,6 +4985,37 @@ class _ManagedFileTableOverRows:
|
|||
rows: Final = ordered_rows[start:]
|
||||
return rows if take is None else rows[:take]
|
||||
|
||||
async def query_raw(self, query, *params):
|
||||
keys_index: Final = next(index for index, param in enumerate(params) if isinstance(param, list))
|
||||
owner_ids: Final = frozenset(params[:keys_index])
|
||||
model_mapping_keys: Final = frozenset(params[keys_index])
|
||||
cursor_id: Final = params[keys_index + 1] if len(params) == keys_index + 3 else None
|
||||
ordered_rows: Final = [
|
||||
row
|
||||
for row in (self.rows[::-1] if '"created_at" ASC' in query else self.rows)
|
||||
if not owner_ids or {row.created_by, row.team_id} & owner_ids
|
||||
]
|
||||
start: Final = (
|
||||
0
|
||||
if cursor_id is None
|
||||
else 1 + next(index for index, row in enumerate(ordered_rows) if row.unified_file_id == cursor_id)
|
||||
)
|
||||
listed_rows: Final = [
|
||||
row
|
||||
for row in ordered_rows[start:]
|
||||
if not model_mapping_keys.isdisjoint(row.model_mappings)
|
||||
or ('"storage_backend" IS NOT NULL' in query and row.storage_backend is not None)
|
||||
]
|
||||
return [
|
||||
{
|
||||
"unified_file_id": row.unified_file_id,
|
||||
"file_object": row.file_object,
|
||||
"model_mappings": row.model_mappings,
|
||||
"storage_backend": row.storage_backend,
|
||||
}
|
||||
for row in listed_rows[: params[-1]]
|
||||
]
|
||||
|
||||
|
||||
def _setup_unscoped_list_files_route_over_real_hook(
|
||||
mocker, monkeypatch, llm_router: Router, rows
|
||||
|
|
@ -5006,7 +5037,9 @@ def _setup_unscoped_list_files_route_over_real_hook(
|
|||
managed_files = PROXY_LiteLLMManagedFiles(
|
||||
internal_usage_cache=MagicMock(), prisma_client=MagicMock()
|
||||
)
|
||||
managed_files.prisma_client.db.litellm_managedfiletable = _ManagedFileTableOverRows(rows)
|
||||
managed_file_table = _ManagedFileTableOverRows(rows)
|
||||
managed_files.prisma_client.db.litellm_managedfiletable = managed_file_table
|
||||
managed_files.prisma_client.db.query_raw = managed_file_table.query_raw
|
||||
|
||||
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, llm_router)
|
||||
proxy_logging_obj.proxy_hook_mapping["managed_files"] = managed_files
|
||||
|
|
@ -6284,6 +6317,37 @@ def test_require_managed_files_lists_the_callers_files_on_the_named_provider_or_
|
|||
provider_list.assert_not_awaited()
|
||||
|
||||
|
||||
def test_require_managed_files_lists_a_teammates_files_on_the_provider_and_no_other_teams(
|
||||
mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch, llm_router: Router
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
_setup_unscoped_list_files_route_over_real_hook(
|
||||
mocker,
|
||||
monkeypatch,
|
||||
llm_router,
|
||||
[
|
||||
_managed_file_row(
|
||||
"unified-teammate", created_by="teammate", team_id="team-a", model_mappings={"gpt-3.5-turbo-id": "f-1"}
|
||||
),
|
||||
_managed_file_row("unified-own", model_mappings={"gpt-3.5-turbo-id": "f-2"}),
|
||||
_managed_file_row(
|
||||
"unified-other-team", created_by="outsider", team_id="team-b", model_mappings={"gpt-3.5-turbo-id": "f-3"}
|
||||
),
|
||||
],
|
||||
)
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
api_key="test-key", user_role=LitellmUserRoles.INTERNAL_USER, user_id="test-user", team_id="team-a"
|
||||
)
|
||||
monkeypatch.setattr("litellm.require_managed_files", True)
|
||||
|
||||
response: Final = _list_files_as_test_user("/openai/v1/files", {}, {})
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert [file["id"] for file in response.json()["data"]] == ["unified-teammate", "unified-own"]
|
||||
|
||||
|
||||
def _router_with_wildcard_and_unprefixed_anthropic_deployments() -> Router:
|
||||
unprefixed_anthropic_model: Final = next(model for model in sorted(litellm.anthropic_models) if "/" not in model)
|
||||
return Router(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue