From 069ded7b6778be7b51fd89dec2228ac41bc8e871 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:48:23 +0000 Subject: [PATCH] refactor(redis): isolate managed file metadata store --- .../proxy/hooks/managed_files.py | 354 +++---- litellm/redis/stores/managed_files.py | 65 ++ .../proxy/hooks/test_managed_files.py | 903 +++++------------- .../proxy/test_managed_files_hook.py | 197 ++-- tests/unit/redis/stores/test_managed_files.py | 264 +++++ 5 files changed, 845 insertions(+), 938 deletions(-) create mode 100644 litellm/redis/stores/managed_files.py create mode 100644 tests/unit/redis/stores/test_managed_files.py diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 5b54e7d72a0..f5ac8718be6 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -4,19 +4,18 @@ import asyncio import base64 import json +import math import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timedelta from itertools import chain from types import MappingProxyType from typing import ( TYPE_CHECKING, Any, - Dict, Final, - List, Literal, - Optional, Protocol, TypedDict, Union, @@ -36,6 +35,8 @@ from litellm import Router, verbose_logger from litellm._internal_context import with_service_target from litellm._uuid import uuid from litellm.caching.caching import DualCache +from litellm.caching.redis_batch import active_request_redis_batch +from litellm.caching.redis_cache import RedisCache from litellm.constants import ( BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS, BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS, @@ -93,6 +94,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( request_tags_from_metadata, ) +from litellm.redis.stores.managed_files import ManagedFileStore, TransportManagedFileStore, file_key +from litellm.redis.transport.contract import RedisFailure from litellm.repositories.managed_file_repository import ManagedFileRepository from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue] @@ -158,7 +161,7 @@ def _decode_json_blob(blob: object) -> object: return json.loads(blob) if isinstance(blob, str) else blob -def _parse_managed_batch_row(row: "PrismaManagedObjectRow") -> Optional[LiteLLMBatch]: +def _parse_managed_batch_row(row: "PrismaManagedObjectRow") -> LiteLLMBatch | None: try: batch_obj: Final = LiteLLMBatch.model_validate(_decode_json_blob(row.file_object)) except Exception as e: @@ -168,7 +171,7 @@ def _parse_managed_batch_row(row: "PrismaManagedObjectRow") -> Optional[LiteLLMB return batch_obj -def _parse_managed_file_object(raw_file_object: object, unified_file_id: str) -> Optional[OpenAIFileObject]: +def _parse_managed_file_object(raw_file_object: object, unified_file_id: str) -> OpenAIFileObject | None: if raw_file_object is None: return None try: @@ -311,16 +314,16 @@ class _ManagedFileRow(Protocol): file_object: OpenAIFileObject model_mappings: Mapping[str, str] flat_model_file_ids: Sequence[str] - storage_backend: Optional[str] - storage_url: Optional[str] - created_by: Optional[str] - team_id: Optional[str] + storage_backend: str | None + storage_url: str | None + created_by: str | None + team_id: str | None def model_dump(self) -> Mapping[str, object]: ... class _ManagedFileTableActions(Protocol): - async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ... + async def find_first(self, where: Mapping[str, object]) -> _ManagedFileRow | None: ... async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ... @@ -328,24 +331,24 @@ class _ManagedFileTableActions(Protocol): self, where: Mapping[str, object], take: int = ..., - order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ..., + order: Mapping[str, str] | Sequence[Mapping[str, str]] = ..., cursor: Mapping[str, str] = ..., skip: int = ..., ) -> Sequence[_ManagedFileRow]: ... async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ... - async def delete(self, where: Mapping[str, str]) -> Optional[_ManagedFileRow]: ... + async def delete(self, where: Mapping[str, str]) -> _ManagedFileRow | None: ... class _ManagedObjectTableActions(Protocol): - async def find_first(self, where: Mapping[str, object]) -> "Optional[PrismaManagedObjectRow]": ... + async def find_first(self, where: Mapping[str, object]) -> "PrismaManagedObjectRow | None": ... async def find_many( self, where: Mapping[str, object], take: int, - order: Union[Mapping[str, str], Sequence[Mapping[str, str]]], + order: Mapping[str, str] | Sequence[Mapping[str, str]], cursor: Mapping[str, str] = ..., skip: int = ..., ) -> "Sequence[PrismaManagedObjectRow]": ... @@ -517,11 +520,7 @@ def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, s "Mapping[str, object]", getattr(file_object, "_hidden_params", None) or {} ) return MappingProxyType( - { - key: value - for key in ("storage_backend", "storage_url") - if isinstance(value := hidden_params.get(key), str) - } + {key: value for key in ("storage_backend", "storage_url") if isinstance(value := hidden_params.get(key), str)} ) @@ -578,6 +577,19 @@ def _keyed_get(source: object, key: str, default: object = None) -> object: return require_attributes(source, _SupportsKeyedGet, "get").get(key, default) +class _LocalMetadataCache(Protocol): + async def async_get_cache(self, key: str, *, local_only: bool) -> object: ... + + async def async_set_cache(self, key: str, value: object, *, local_only: bool) -> None: ... + + +@dataclass(frozen=True, slots=True) +class _MetadataCaches: + local: _LocalMetadataCache + redis: RedisCache | None + in_memory_ttl: float | None + + class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Class variables or attributes def __init__( @@ -593,6 +605,56 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self._sleep = sleep self._clock = clock + def _metadata_cache(self) -> _MetadataCaches: + cache: Final = self.internal_usage_cache + dual: Final = cache if isinstance(cache, DualCache) else getattr(cache, "dual_cache", None) + if isinstance(dual, DualCache): + return _MetadataCaches(local=dual, redis=dual.redis_cache, in_memory_ttl=dual.default_in_memory_ttl) + return _MetadataCaches( + local=require_attributes(cache, _LocalMetadataCache, "async_get_cache", "async_set_cache"), + redis=None, + in_memory_ttl=None, + ) + + async def _cache_file(self, file_id: str, record: Mapping[str, object] | None) -> None: + cache: Final = self._metadata_cache() + local_cache: Final = cache.local + await local_cache.async_set_cache(file_key(file_id), record, local_only=True) + if cache.redis is None: + return + store: Final[ManagedFileStore] = TransportManagedFileStore(cache.redis.transport()) + try: + seconds: Final = ( + int(cache.in_memory_ttl) + if cache.in_memory_ttl is not None and not math.isnan(cache.in_memory_ttl) + else cache.redis.default_ttl + ) + ttl: Final = timedelta(seconds=seconds) + except (OverflowError, ValueError): + return + if record is None: + await store.invalidate_file(file_id, ttl=ttl) + else: + await store.write_file(file_id, record, ttl=ttl) + + async def _cached_file(self, file_id: str) -> object: + cache: Final = self._metadata_cache() + local_cache: Final = cache.local + local: Final = await local_cache.async_get_cache(file_key(file_id), local_only=True) + if local is not None or cache.redis is None: + return local + request_batch: Final = active_request_redis_batch(cache.redis) + if request_batch is not None and request_batch.read_as_missing(file_key(file_id)): + return None + store: Final[ManagedFileStore] = TransportManagedFileStore(cache.redis.transport()) + result: Final = await store.read_file(file_id) + if result is None or isinstance(result, RedisFailure): + return None + value: Final = result.value + if value is not None: + await local_cache.async_set_cache(file_key(file_id), value, local_only=True) + return value + @staticmethod def _get_prometheus_logger(): """Find PrometheusLogger from litellm.callbacks, if registered.""" @@ -604,9 +666,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def store_unified_file_id( self, file_id: str, - file_object: Optional[OpenAIFileObject], - litellm_parent_otel_span: Optional[Span], - model_mappings: Dict[str, str], + file_object: OpenAIFileObject | None, + litellm_parent_otel_span: Span | None, + model_mappings: dict[str, str], user_api_key_dict: UserAPIKeyAuth, ) -> None: verbose_logger.info(f"Storing LiteLLM Managed File object with id={file_id} in cache") @@ -623,11 +685,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): storage_backend=storage_metadata.get("storage_backend"), storage_url=storage_metadata.get("storage_url"), ) - await self.internal_usage_cache.async_set_cache( - key=file_id, - value=litellm_managed_file_object.model_dump(), - litellm_parent_otel_span=litellm_parent_otel_span, - ) + await self._cache_file(file_id, litellm_managed_file_object.model_dump()) ## STORE MODEL MAPPINGS IN DB @@ -663,7 +721,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) verbose_logger.debug(f"LiteLLM Managed File object with id={file_id} stored in db: {result}") - async def _resolve_creator_org_id(self, user_api_key_dict: UserAPIKeyAuth) -> Optional[str]: + async def _resolve_creator_org_id(self, user_api_key_dict: UserAPIKeyAuth) -> str | None: if user_api_key_dict.org_id: return user_api_key_dict.org_id if not user_api_key_dict.team_id: @@ -689,7 +747,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self, unified_object_id: str, file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob, "ResponsesAPIResponse"], - litellm_parent_otel_span: Optional[Span], + litellm_parent_otel_span: Span | None, model_object_id: str, file_purpose: Literal["batch", "fine-tune", "response"], user_api_key_dict: UserAPIKeyAuth, @@ -817,16 +875,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): @with_service_target(_MANAGED_FILES_TARGET) async def get_unified_file_id( - self, file_id: str, litellm_parent_otel_span: Optional[Span] = None - ) -> Optional[LiteLLM_ManagedFileTable]: + self, file_id: str, litellm_parent_otel_span: Span | None = None + ) -> LiteLLM_ManagedFileTable | None: ## CHECK CACHE - result = cast( - Optional[dict], - await self.internal_usage_cache.async_get_cache( - key=file_id, - litellm_parent_otel_span=litellm_parent_otel_span, - ), - ) + result: Final = await self._cached_file(file_id) if result: return LiteLLM_ManagedFileTable.model_validate(result) @@ -840,18 +892,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): @with_service_target(_MANAGED_FILES_TARGET) async def delete_unified_file_id( - self, file_id: str, litellm_parent_otel_span: Optional[Span] = None + self, file_id: str, litellm_parent_otel_span: Span | None = None ) -> OpenAIFileObject: ## get old value initial_value = await _managed_file_table(self.prisma_client).find_first(where={"unified_file_id": file_id}) if initial_value is None: raise Exception(f"LiteLLM Managed File object with id={file_id} not found") ## delete old value - await self.internal_usage_cache.async_set_cache( - key=file_id, - value=None, - litellm_parent_otel_span=litellm_parent_otel_span, - ) + await self._cache_file(file_id, None) await _managed_file_table(self.prisma_client).delete(where={"unified_file_id": file_id}) return initial_value.file_object @@ -887,9 +935,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): detail=f"Object not found: {unified_object_id}", ) - async def enforce_batch_object_access( - self, object_id: str, user_api_key_dict: UserAPIKeyAuth - ) -> None: + async def enforce_batch_object_access(self, object_id: str, user_api_key_dict: UserAPIKeyAuth) -> None: """Deny access to a provider-format batch id owned by another caller. Ids with no ownership row (batches created before ownership tracking, @@ -913,9 +959,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): detail=f"User {user_api_key_dict.user_id} does not have access to the object {object_id}", ) - async def enforce_provider_file_access( - self, file_id: str, user_api_key_dict: UserAPIKeyAuth - ) -> None: + async def enforce_provider_file_access(self, file_id: str, user_api_key_dict: UserAPIKeyAuth) -> None: """Deny access to a provider-format file id owned by another caller. Ownership rows for provider-format ids are written when a managed @@ -940,7 +984,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) async def store_batch_output_file_ownership( - self, response: LiteLLMBatch, litellm_parent_otel_span: Optional[Span] + self, response: LiteLLMBatch, litellm_parent_otel_span: Span | None ) -> None: """Record ownership rows for a batch's provider-format output/error file ids, inherited from the owning batch row (never the caller), so @@ -957,16 +1001,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return if self.prisma_client is None: return - batch_row = await _managed_object_table(self.prisma_client).find_first( - where={"unified_object_id": response.id} - ) - if batch_row is None or ( - batch_row.created_by is None and batch_row.team_id is None - ): + batch_row = await _managed_object_table(self.prisma_client).find_first(where={"unified_object_id": response.id}) + if batch_row is None or (batch_row.created_by is None and batch_row.team_id is None): return - owner_identity = UserAPIKeyAuth( - user_id=batch_row.created_by, team_id=batch_row.team_id - ) + owner_identity = UserAPIKeyAuth(user_id=batch_row.created_by, team_id=batch_row.team_id) for file_id in provider_file_ids: model_name = decode_model_from_file_id(file_id) raw_file_id = get_original_file_id(file_id) @@ -1084,8 +1122,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if raise_on_failure: if isinstance(error, TimeoutError) and not str(error): raise TimeoutError( - "Provider file retrieve timed out " - f"after {BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS} seconds" + f"Provider file retrieve timed out after {BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS} seconds" ) from error raise return None, True @@ -1098,11 +1135,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if not await ManagedFileRepository(self.prisma_client).update_file_object(stored.unified_file_id, file_object): return refreshed_row: Final = stored.model_copy(update={"file_object": file_object}) - await self.internal_usage_cache.async_set_cache( - key=stored.unified_file_id, - value=refreshed_row.model_dump(), - litellm_parent_otel_span=None, - ) + await self._cache_file(stored.unified_file_id, refreshed_row.model_dump()) async def _save_xai_refreshed_file_object( self, @@ -1117,14 +1150,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): sized_file_object, if_file_object=_stored_json_text(stored.file_object) if stored.file_object is not None else None, ): - await self.internal_usage_cache.async_set_cache( - key=stored.unified_file_id, value=None, litellm_parent_otel_span=None - ) + await self._cache_file(stored.unified_file_id, None) return None - await self.internal_usage_cache.async_set_cache( - key=stored.unified_file_id, - value=stored.model_copy(update={"file_object": sized_file_object}).model_dump(), - litellm_parent_otel_span=None, + await self._cache_file( + stored.unified_file_id, stored.model_copy(update={"file_object": sized_file_object}).model_dump() ) return sized_file_object @@ -1270,12 +1299,12 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def list_user_batches( self, user_api_key_dict: UserAPIKeyAuth, - limit: Optional[int] = None, - after: Optional[str] = None, - provider: Optional[str] = None, - target_model_names: Optional[str] = None, - llm_router: Optional[Router] = None, - ) -> Dict[str, object]: + limit: int | None = None, + after: str | None = None, + provider: str | None = None, + target_model_names: str | None = None, + llm_router: Router | None = None, + ) -> dict[str, object]: # Provider filtering is not supported for managed batches # This is because the encoded object ids stored in the managed objects table do not contain the provider information # To support provider filtering, we would need to store the provider information in the encoded object ids @@ -1305,7 +1334,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if owner_filter is None: return build_list_page([]) - where_clause: Dict[str, object] = {"file_purpose": "batch", **owner_filter} + where_clause: dict[str, object] = {"file_purpose": "batch", **owner_filter} if after: cursor_row = await _managed_object_table(self.prisma_client).find_first( @@ -1329,7 +1358,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def _collect_listed_batches( self, where_clause: Mapping[str, object], - after: Optional[str], + after: str | None, wanted: int, user_api_key_dict: UserAPIKeyAuth, ) -> tuple[LiteLLMBatch, ...]: @@ -1341,7 +1370,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): and every chunk advances the keyset cursor, so the walk ends once the caller's rows are exhausted.""" matches: tuple[LiteLLMBatch, ...] = () # rebind-ok: accumulates survivors across chunks - cursor_id: Optional[str] = after # rebind-ok: keyset cursor advances to each chunk's last row + cursor_id: str | None = after # rebind-ok: keyset cursor advances to each chunk's last row chunk_size: int = wanted # rebind-ok: widens once a scan has to continue past the first chunk while len(matches) < wanted: cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": cursor_id}, "skip": 1} if cursor_id else {} @@ -1398,7 +1427,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): batch_obj: LiteLLMBatch, unified_id_by_raw_id: Mapping[str, str], user_api_key_dict: UserAPIKeyAuth, - ) -> Optional[LiteLLMBatch]: + ) -> LiteLLMBatch | None: apply_unified_file_ids(batch_obj, unified_id_by_raw_id) try: await ensure_batch_response_managed_file_ids( @@ -1450,8 +1479,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) async def get_user_created_file_ids( - self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str] - ) -> List[OpenAIFileObject]: + self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: list[str] + ) -> list[OpenAIFileObject]: """ Get all file ids the caller is allowed to see for a list of model object ids. Service-account keys (no user_id) are scoped to their @@ -1476,8 +1505,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None ] - async def check_managed_file_id_access(self, data: Dict, user_api_key_dict: UserAPIKeyAuth) -> bool: - retrieve_file_id = cast(Optional[str], data.get("file_id")) + async def check_managed_file_id_access(self, data: dict, user_api_key_dict: UserAPIKeyAuth) -> bool: + retrieve_file_id = cast(str | None, data.get("file_id")) potential_file_id = _is_base64_encoded_unified_file_id(retrieve_file_id) if retrieve_file_id else False if potential_file_id and retrieve_file_id: if await self.can_user_call_unified_file_id(retrieve_file_id, user_api_key_dict): @@ -1488,9 +1517,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}", ) if retrieve_file_id: - await self.enforce_provider_file_access( - retrieve_file_id, user_api_key_dict - ) + await self.enforce_provider_file_access(retrieve_file_id, user_api_key_dict) return False async def check_file_ids_access(self, file_ids: Sequence[str], user_api_key_dict: UserAPIKeyAuth) -> None: @@ -1518,9 +1545,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, - data: Dict, + data: dict, call_type: CallTypesLiteral, - ) -> Union[Exception, str, Dict, None]: + ) -> Exception | str | dict | None: """ - Detect litellm_proxy/ file_id - add dictionary of mappings of litellm_proxy/ file_id -> provider_file_id => {litellm_proxy/file_id: {"model_id": id, "file_id": provider_file_id}} @@ -1593,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if _is_streamed_background_create(create_request): return _with_managed_background_marker(create_request) elif call_type == CallTypes.afile_content.value: - retrieve_file_id = cast(Optional[str], data.get("file_id")) + retrieve_file_id = cast(str | None, data.get("file_id")) potential_file_id = _is_base64_encoded_unified_file_id(retrieve_file_id) if retrieve_file_id else False if potential_file_id and "llm_output_file_id," in potential_file_id: model_id = self.get_model_id_from_unified_file_id(potential_file_id) @@ -1601,7 +1628,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): data["model"] = model_id data["file_id"] = self.get_output_file_id_from_unified_file_id(potential_file_id) elif call_type == CallTypes.acreate_batch.value: - input_file_id = cast(Optional[str], data.get("input_file_id")) + input_file_id = cast(str | None, data.get("input_file_id")) if input_file_id: await self.check_file_ids_access((input_file_id,), user_api_key_dict) model_file_id_mapping = await self.get_model_file_id_mapping( @@ -1615,8 +1642,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): or call_type == CallTypes.acancel_fine_tuning_job.value or call_type == CallTypes.aretrieve_fine_tuning_job.value ): - accessor_key: Optional[str] = None - retrieve_object_id: Optional[str] = None + accessor_key: str | None = None + retrieve_object_id: str | None = None if call_type == CallTypes.aretrieve_batch.value or call_type == CallTypes.acancel_batch.value: accessor_key = "batch_id" elif ( @@ -1626,7 +1653,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): accessor_key = "fine_tuning_job_id" if accessor_key: - retrieve_object_id = cast(Optional[str], data.get(accessor_key)) + retrieve_object_id = cast(str | None, data.get(accessor_key)) potential_llm_object_id = ( _is_base64_encoded_unified_file_id(retrieve_object_id) if retrieve_object_id else False @@ -1670,11 +1697,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def async_filter_deployments( self, model: str, - healthy_deployments: List, - messages: Optional[List[AllMessageValues]], - request_kwargs: Optional[Dict] = None, - parent_otel_span: Optional[Span] = None, - ) -> List[Dict]: + healthy_deployments: list, + messages: list[AllMessageValues] | None, + request_kwargs: dict | None = None, + parent_otel_span: Span | None = None, + ) -> list[dict]: if request_kwargs is None: return healthy_deployments @@ -1701,8 +1728,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return holders if holders or "input_file_id" in request else healthy_deployments async def async_pre_call_deployment_hook( - self, kwargs: dict[str, object], call_type: Optional[CallTypes] - ) -> Optional[dict]: + self, kwargs: dict[str, object], call_type: CallTypes | None + ) -> dict | None: """ Allow modifying the request just before it's sent to the deployment. """ @@ -1710,18 +1737,18 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if accessor_keys is None: return kwargs - model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping")) + model_file_id_mapping = cast(dict[str, dict[str, str]] | None, kwargs.get("model_file_id_mapping")) # model_info may be at top-level or nested under litellm_metadata # (batch/file operations use litellm_metadata) - model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id")) + model_id = cast(str | None, _keyed_get(kwargs.get("model_info", {}), "id")) if model_id is None: model_id = cast( - Optional[str], + str | None, _keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"), ) for accessor_key in accessor_keys: - input_file_id = cast(Optional[str], kwargs.get(accessor_key)) - mapped_file_id: Optional[str] = None + input_file_id = cast(str | None, kwargs.get(accessor_key)) + mapped_file_id: str | None = None if input_file_id and model_file_id_mapping and model_id: mapped_file_id = model_file_id_mapping.get(input_file_id, {}).get(model_id, None) if mapped_file_id: @@ -1729,7 +1756,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return kwargs - def get_file_ids_from_messages(self, messages: List[AllMessageValues]) -> List[str]: + def get_file_ids_from_messages(self, messages: list[AllMessageValues]) -> list[str]: """ Gets file ids from messages """ @@ -1749,7 +1776,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_ids.append(file_id) return file_ids - def get_file_ids_from_responses_input(self, input: Union[str, List[Dict[str, object]]]) -> List[str]: + def get_file_ids_from_responses_input(self, input: str | list[dict[str, object]]) -> list[str]: """ Gets file ids from responses API input. @@ -1759,7 +1786,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): - type: "input_file" with file_id - content: a list that can contain items with type: "input_file" and file_id """ - file_ids: List[str] = [] + file_ids: list[str] = [] if isinstance(input, str): return file_ids @@ -1788,7 +1815,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return file_ids - def get_file_ids_from_responses_tools(self, tools: List[Dict[str, object]]) -> List[str]: + def get_file_ids_from_responses_tools(self, tools: list[dict[str, object]]) -> list[str]: """ Gets file ids from responses API tools parameter. @@ -1800,7 +1827,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): } ] """ - file_ids: List[str] = [] + file_ids: list[str] = [] if not isinstance(tools, list): return file_ids @@ -1821,7 +1848,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return file_ids - def get_vector_store_ids_from_file_search_tools(self, tools: List[Dict[str, object]]) -> List[str]: + def get_vector_store_ids_from_file_search_tools(self, tools: list[dict[str, object]]) -> list[str]: """ Extract unified vector_store_ids from file_search tools. @@ -1832,7 +1859,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): is_base64_encoded_unified_id, ) - vs_ids: List[str] = [] + vs_ids: list[str] = [] if not isinstance(tools, list): return vs_ids @@ -1850,7 +1877,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def check_vector_store_ids_access( self, - vector_store_ids: List[str], + vector_store_ids: list[str], user_api_key_dict: UserAPIKeyAuth, ) -> None: """ @@ -1876,7 +1903,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return # Map each unified ID to its internal UUID for a single batch DB fetch - uuid_to_unified: Dict[str, str] = {} + uuid_to_unified: dict[str, str] = {} for vs_id in vector_store_ids: uuid = extract_unified_uuid_from_unified_id(vs_id) if uuid: @@ -1931,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): } """ - file_id_mapping: Dict[str, Dict[str, str]] = {} + file_id_mapping: dict[str, dict[str, str]] = {} litellm_managed_file_ids = [] for file_id in file_ids: @@ -1953,11 +1980,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def create_file_for_each_model( self, - llm_router: Optional[Router], + llm_router: Router | None, _create_file_request: CreateFileRequest, - target_model_names_list: List[str], + target_model_names_list: list[str], litellm_parent_otel_span: Span | None, - ) -> List[OpenAIFileObject]: + ) -> list[OpenAIFileObject]: if llm_router is None: raise Exception("LLM Router not initialized. Ensure models added to proxy.") responses = [] @@ -1971,7 +1998,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self, create_file_request: CreateFileRequest, llm_router: Router, - target_model_names_list: List[str], + target_model_names_list: list[str], litellm_parent_otel_span: Span | None, user_api_key_dict: UserAPIKeyAuth, ) -> OpenAIFileObject: @@ -1990,7 +2017,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) ## STORE MODEL MAPPINGS IN DB - model_mappings: Dict[str, str] = {} + model_mappings: dict[str, str] = {} for file_object in responses: file_hidden_params = cast( # cast-ok: preserve mapping operations on dynamic file metadata @@ -2036,11 +2063,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): @staticmethod async def return_unified_file_id( - file_objects: List[OpenAIFileObject], + file_objects: list[OpenAIFileObject], create_file_request: CreateFileRequest, internal_usage_cache: InternalUsageCache, litellm_parent_otel_span: Span | None, - target_model_names_list: List[str], + target_model_names_list: list[str], ) -> OpenAIFileObject: ## GET THE FILE TYPE FROM THE CREATE FILE REQUEST _, file_type = extract_file_metadata(create_file_request["file"]) @@ -2085,7 +2112,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): unified_batch_id = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format(model_id, batch_id) return base64.urlsafe_b64encode(unified_batch_id.encode()).decode().rstrip("=") - def get_unified_output_file_id(self, output_file_id: str, model_id: str, model_name: Optional[str]) -> str: + def get_unified_output_file_id(self, output_file_id: str, model_id: str, model_name: str | None) -> str: deterministic_uuid: Final = uuid5(uuid5(NAMESPACE_URL, model_id), output_file_id) unified_output_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( "application/json", @@ -2106,7 +2133,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return file_id.split(marker, 1)[1].split(";")[0] async def async_post_call_success_hook( - self, data: Dict, user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes + self, data: dict, user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes ) -> LLMResponseTypes: if isinstance(response, LiteLLMBatch): decoded_batch_id: Final = _is_base64_encoded_unified_file_id(response.id) @@ -2117,8 +2144,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): unified_file_id = response_hidden_params.get("unified_file_id") unified_batch_id = response_hidden_params.get("unified_batch_id") is_batch_create: Final = response_hidden_params.get(BATCH_CREATE_HIDDEN_PARAM) is True - model_id = cast(Optional[str], response_hidden_params.get("model_id")) - model_name = cast(Optional[str], response_hidden_params.get("model_name")) + model_id = cast(str | None, response_hidden_params.get("model_id")) + model_name = cast(str | None, response_hidden_params.get("model_name")) resolved_model_name = resolve_managed_output_file_model_name( unified_input_file_id=unified_file_id if isinstance(unified_file_id, str) else response.input_file_id, @@ -2206,7 +2233,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) unified_file_id = finetuning_response_hidden_params.get("unified_file_id") unified_finetuning_job_id = finetuning_response_hidden_params.get("unified_finetuning_job_id") - model_id = cast(Optional[str], finetuning_response_hidden_params.get("model_id")) + model_id = cast(str | None, finetuning_response_hidden_params.get("model_id")) original_response_id = response.id if (unified_file_id or unified_finetuning_job_id) and model_id: response.id = self.get_unified_generic_response_id(model_id=model_id, generic_response_id=response.id) @@ -2224,15 +2251,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ ## check if file object if hasattr(response, "data") and isinstance(response.data, list): - if all(isinstance(file_object, FileObject) for file_object in response.data): - ## Get all file id's - ## Check which file id's were created by the user - ## Filter the response to only include the files created by the user - ## Return the filtered response - file_ids = [ - file_object.id - for file_object in cast(List[FileObject], response.data) # type: ignore - ] + page_items: Final = TypeAdapter(tuple[object, ...]).validate_python(response.data) + file_objects: Final = tuple(item for item in page_items if isinstance(item, FileObject)) + if len(file_objects) == len(page_items): + file_ids = [file_object.id for file_object in file_objects] user_created_file_ids = await self.get_user_created_file_ids(user_api_key_dict, file_ids) ## Filter the response to only include the files created by the user response.data = user_created_file_ids # type: ignore @@ -2242,7 +2264,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return response @staticmethod - def _scope_list_page_cursors(response: AsyncCursorPage, data: List[OpenAIFileObject]) -> None: + def _scope_list_page_cursors(response: AsyncCursorPage, data: list[OpenAIFileObject]) -> None: """Rebuild ``first_id`` / ``last_id`` from the caller-scoped page. The upstream cursors point at rows that were just filtered out, so @@ -2258,7 +2280,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): response.has_more = False async def afile_retrieve( - self, file_id: str, litellm_parent_otel_span: Optional[Span], llm_router: Optional[Router] = None + self, file_id: str, litellm_parent_otel_span: Span | None, llm_router: Router | None = None ) -> OpenAIFileObject: """Return public details for a managed file ID, refreshing a basic entry when possible.""" stored_file_object = await self.get_unified_file_id(file_id, litellm_parent_otel_span) @@ -2325,7 +2347,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): allow_default_provider=True, ) except Exception as e: - raise Exception(f"Failed to retrieve file {file_id} from provider: {str(e)}") from e + raise Exception(f"Failed to retrieve file {file_id} from provider: {e!s}") from e if not fetch_route_available: raise Exception( f"LiteLLM Managed File object with id={file_id} has no file_object and no provider route to fetch it" @@ -2336,14 +2358,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def afile_list( self, - purpose: Optional[str], - litellm_parent_otel_span: Optional[Span], + purpose: str | None, + litellm_parent_otel_span: Span | None, user_api_key_dict: UserAPIKeyAuth, - limit: Optional[int] = None, - after: Optional[str] = None, - model_scope: Optional[FileListModelScope] = None, - before: Optional[str] = None, - **data: Dict, + limit: int | None = None, + after: str | None = None, + model_scope: FileListModelScope | None = None, + before: str | None = None, + **data: dict, ) -> FileListPage: """List the managed files the caller owns, newest first. @@ -2404,7 +2426,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return FileListPage.model_validate(build_list_page([])) page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT) - matches: Final[List[OpenAIFileObject]] = [] + matches: Final[list[OpenAIFileObject]] = [] cursor_id = cursor chunk_size = page_size + 1 direction: Final[Literal["asc", "desc"]] = "asc" if before else "desc" @@ -2471,7 +2493,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): verbose_logger.warning(f"Error checking batch polling configuration: {e}. Assuming disabled.") return False - async def _get_batches_referencing_file(self, file_id: str) -> List[Dict[str, object]]: + async def _get_batches_referencing_file(self, file_id: str) -> list[dict[str, object]]: """ Find batches that reference this file and still need cost tracking. Find batches that are in non-terminal state and have not yet been processed by CheckBatchCost. @@ -2607,9 +2629,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def afile_delete( self, file_id: str, - litellm_parent_otel_span: Optional[Span], + litellm_parent_otel_span: Span | None, llm_router: Router, - **data: Dict, + **data: dict, ) -> FileDeleted: # Check if file deletion should be blocked due to batch references @@ -2668,9 +2690,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def afile_content( self, file_id: str, - litellm_parent_otel_span: Optional[Span], + litellm_parent_otel_span: Span | None, llm_router: Router, - **data: Dict, + **data: dict, ) -> HttpxBinaryResponseContent: """ Get the content of a file from first model that has it @@ -2696,7 +2718,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # request params. credentials = llm_router.registry().credentials(model_id, form="with_provider") if credentials is not None: - data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials))) + data["_litellm_internal_model_credentials"] = cast(dict, MappingProxyType(dict(credentials))) else: data.pop("_litellm_internal_model_credentials", None) return cast( @@ -2722,9 +2744,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def _convert_storage_files_to_base64( self, - messages: List[AllMessageValues], - file_ids: List[str], - litellm_parent_otel_span: Optional[Span], + messages: list[AllMessageValues], + file_ids: list[str], + litellm_parent_otel_span: Span | None, ) -> None: """ Convert files stored in storage backends to base64 format for Vertex AI/Gemini. @@ -2757,9 +2779,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): try: storage_backend = get_storage_backend(storage_backend_name, prisma_client=self.prisma_client) except ValueError as e: - verbose_logger.warning( - f"Storage backend '{storage_backend_name}' error for file {file_id}: {str(e)}" - ) + verbose_logger.warning(f"Storage backend '{storage_backend_name}' error for file {file_id}: {e!s}") continue file_content = await storage_backend.download_file(storage_url) @@ -2774,7 +2794,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Update messages to use base64 instead of file_id self._update_messages_with_base64_data(messages, file_id, base64_data_uri, content_type) except Exception as e: - verbose_logger.exception(f"Error converting file {file_id} from storage backend to base64: {str(e)}") + verbose_logger.exception(f"Error converting file {file_id} from storage backend to base64: {e!s}") # Continue with other files even if one fails continue @@ -2800,7 +2820,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): def _update_messages_with_base64_data( self, - messages: List[AllMessageValues], + messages: list[AllMessageValues], file_id: str, base64_data_uri: str, content_type: str, @@ -2834,4 +2854,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): verbose_logger.debug( f"Converted file {file_id} from storage backend to base64 with format {content_type}" ) + + PROXY_LiteLLMManagedFiles = _PROXY_LiteLLMManagedFiles diff --git a/litellm/redis/stores/managed_files.py b/litellm/redis/stores/managed_files.py new file mode 100644 index 00000000000..d2e7d85fcc5 --- /dev/null +++ b/litellm/redis/stores/managed_files.py @@ -0,0 +1,65 @@ +import ast +import json +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import timedelta +from typing import Final, Protocol + +from pydantic import TypeAdapter + +from litellm.redis.transport.contract import RedisFailure, RedisTransport + + +@dataclass(frozen=True, slots=True) +class ManagedFileValue: + value: object + + +_VALUE: Final[TypeAdapter[ManagedFileValue]] = TypeAdapter(ManagedFileValue) + + +def file_key(unified_file_id: str) -> str: + return unified_file_id + + +class ManagedFileStore(Protocol): + async def read_file(self, unified_file_id: str) -> ManagedFileValue | None | RedisFailure: ... + + async def write_file( + self, unified_file_id: str, record: Mapping[str, object], *, ttl: timedelta | None + ) -> None | RedisFailure: ... + + async def invalidate_file(self, unified_file_id: str, *, ttl: timedelta | None) -> None | RedisFailure: ... + + +@dataclass(frozen=True, slots=True) +class TransportManagedFileStore: + transport: RedisTransport + + async def read_file(self, unified_file_id: str) -> ManagedFileValue | None | RedisFailure: + raw: Final = await self.transport.get(file_key(unified_file_id)) + if raw is None or isinstance(raw, RedisFailure): + return raw + try: + return _VALUE.validate_python({"value": json.loads(raw.decode("utf-8"))}) + except (ValueError, UnicodeError, RecursionError): + try: + return _VALUE.validate_python({"value": ast.literal_eval(raw.decode("utf-8"))}) + except (ValueError, SyntaxError, UnicodeError, RecursionError) as exc: + return RedisFailure(kind="decode", message=str(exc)) + + async def write_file( + self, unified_file_id: str, record: Mapping[str, object], *, ttl: timedelta | None + ) -> None | RedisFailure: + try: + encoded: Final = json.dumps(record) + except (TypeError, ValueError) as exc: + return RedisFailure(kind="decode", message=str(exc)) + return await self._write(unified_file_id, encoded, ttl) + + async def invalidate_file(self, unified_file_id: str, *, ttl: timedelta | None) -> None | RedisFailure: + return await self._write(unified_file_id, "null", ttl) + + async def _write(self, unified_file_id: str, encoded: str, ttl: timedelta | None) -> None | RedisFailure: + result: Final = await self.transport.set_seconds(file_key(unified_file_id), encoded, ttl=ttl) + return result if isinstance(result, RedisFailure) else None diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index 1755ab97689..f24e05c6d41 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -1,8 +1,8 @@ +import asyncio import base64 import json import logging import time -import asyncio import uuid from collections.abc import Awaitable, Callable, Mapping from types import SimpleNamespace @@ -23,33 +23,27 @@ from litellm_enterprise.proxy.hooks.managed_files import ( PROXY_LiteLLMManagedFiles, _provider_file_retrieve_credentials, ) + from litellm.caching import DualCache from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable, UserAPIKeyAuth from litellm.proxy.openai_files_endpoints.common_utils import ( BATCH_CREATE_HIDDEN_PARAM, - is_base64_encoded_unified_file_id, encode_file_id_with_model, + is_base64_encoded_unified_file_id, ) +from litellm.proxy.utils import InternalUsageCache from litellm.types.llms.openai import OpenAIFileObject from tests.unit.litellm_core_utils.attribute_forwarder import AttributeForwarder class _InMemoryManagedFileTable: def __init__(self, *rows: LiteLLM_ManagedFileTable) -> None: - self.rows: dict[str, LiteLLM_ManagedFileTable] = { - row.unified_file_id: row for row in rows - } + self.rows: dict[str, LiteLLM_ManagedFileTable] = {row.unified_file_id: row for row in rows} self.find_first_calls: list[Mapping[str, object]] = [] - self.upsert_calls: list[ - tuple[Mapping[str, object], Mapping[str, Mapping[str, object]]] - ] = [] - self.update_many_calls: list[ - tuple[Mapping[str, object], Mapping[str, object]] - ] = [] + self.upsert_calls: list[tuple[Mapping[str, object], Mapping[str, Mapping[str, object]]]] = [] + self.update_many_calls: list[tuple[Mapping[str, object], Mapping[str, object]]] = [] - async def find_first( - self, where: Mapping[str, object] - ) -> LiteLLM_ManagedFileTable | None: + async def find_first(self, where: Mapping[str, object]) -> LiteLLM_ManagedFileTable | None: self.find_first_calls.append(where) unified_file_id: Final = where.get("unified_file_id") if isinstance(unified_file_id, str): @@ -73,11 +67,7 @@ class _InMemoryManagedFileTable: unified_file_id: Final = cast(str, where["unified_file_id"]) previous_row: Final = self.rows.get(unified_file_id) values: Final = { - **( - cast(dict[str, object], previous_row.model_dump()) - if previous_row is not None - else {} - ), + **(cast(dict[str, object], previous_row.model_dump()) if previous_row is not None else {}), **data["update" if previous_row is not None else "create"], } raw_model_mappings: Final = values.get("model_mappings", "{}") @@ -89,9 +79,7 @@ class _InMemoryManagedFileTable: raw_file_object: Final = values.get("file_object") file_object: Final = cast( dict[str, object] | None, - json.loads(raw_file_object) - if isinstance(raw_file_object, str) - else raw_file_object, + json.loads(raw_file_object) if isinstance(raw_file_object, str) else raw_file_object, ) row: Final = LiteLLM_ManagedFileTable.model_validate( { @@ -118,17 +106,11 @@ class _InMemoryManagedFileTable: None if existing_row.file_object is None else existing_row.file_object.model_dump(mode="json") ): return 0 - file_object: Final = OpenAIFileObject.model_validate( - json.loads(cast(str, data["file_object"])) - ) - self.rows[unified_file_id] = existing_row.model_copy( - update={"file_object": file_object} - ) + file_object: Final = OpenAIFileObject.model_validate(json.loads(cast(str, data["file_object"]))) + self.rows[unified_file_id] = existing_row.model_copy(update={"file_object": file_object}) return 1 - async def delete( - self, where: Mapping[str, object] - ) -> LiteLLM_ManagedFileTable | None: + async def delete(self, where: Mapping[str, object]) -> LiteLLM_ManagedFileTable | None: return self.rows.pop(cast(str, where["unified_file_id"]), None) async def find_many( @@ -143,9 +125,7 @@ class _InMemoryManagedObjectTable: async def find_first(self, where: Mapping[str, object]) -> None: return None - async def update_many( - self, where: Mapping[str, object], data: Mapping[str, object] - ) -> int: + async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: return 0 async def upsert( @@ -174,9 +154,7 @@ def _managed_files_with_fake_prisma( ) -> tuple[PROXY_LiteLLMManagedFiles, _InMemoryManagedFileTable]: managed_file_table: Final = _InMemoryManagedFileTable(*rows) prisma_client: Final = _InMemoryManagedFilesPrismaClient(managed_file_table) - managed_files: Final = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client, sleep=sleep - ) + managed_files: Final = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client, sleep=sleep) return managed_files, managed_file_table @@ -209,6 +187,7 @@ def _marked_fallback_file_row( def _batch_response_for_output_listing() -> "LiteLLMBatch": from openai.types.batch import BatchRequestCounts + from litellm.types.utils import LiteLLMBatch response: Final = LiteLLMBatch( @@ -252,9 +231,7 @@ def test_provider_credentials_warning_sanitizes_newlines( ) -> None: model_id: Final = "deployment-123\nforged warning" router: Final = MagicMock() - router.registry.return_value.credentials.side_effect = RuntimeError( - "credential lookup failed\nforged warning" - ) + router.registry.return_value.credentials.side_effect = RuntimeError("credential lookup failed\nforged warning") with caplog.at_level(logging.WARNING, logger="LiteLLM"): credentials: Final = _provider_file_retrieve_credentials( @@ -276,9 +253,7 @@ def test_provider_credentials_warning_sanitizes_newlines( def test_get_file_ids_from_messages(): - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) messages = [ { "role": "user", @@ -300,9 +275,7 @@ def test_get_file_ids_from_messages(): def test_get_file_ids_from_messages_skips_bedrock_content_blocks_without_type(): - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) messages = [ { "role": "user", @@ -340,13 +313,9 @@ async def test_async_pre_call_hook_batch_retrieve(): return_value = MagicMock() return_value.created_by = "123" prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) data = { - "user_api_key_dict": UserAPIKeyAuth( - user_id="123", parent_otel_span=MagicMock() - ), + "user_api_key_dict": UserAPIKeyAuth(user_id="123", parent_otel_span=MagicMock()), "data": { "batch_id": "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1nZW5lcmFsLWF6dXJlLWRlcGxveW1lbnQ7bGxtX2JhdGNoX2lkOmJhdGNoX2EzMjJiNmJhLWFjN2UtNDg4OC05MjljLTFhZDM0NDJmMDZlZA", }, @@ -385,9 +354,7 @@ async def test_async_pre_call_deployment_hook_resolves_model_id_from_litellm_met async_pre_call_deployment_hook must check both locations so the managed file ID is resolved to the provider-specific file ID. """ - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) managed_file_id = "managed-file-abc" model_id = "deployment-xyz" @@ -404,13 +371,11 @@ async def test_async_pre_call_deployment_hook_resolves_model_id_from_litellm_met }, } - result = await proxy_managed_files.async_pre_call_deployment_hook( - kwargs=kwargs, call_type=CallTypes.acreate_batch - ) + result = await proxy_managed_files.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.acreate_batch) - assert ( - result["input_file_id"] == provider_file_id - ), f"Expected provider file ID '{provider_file_id}', got '{result['input_file_id']}'" + assert result["input_file_id"] == provider_file_id, ( + f"Expected provider file ID '{provider_file_id}', got '{result['input_file_id']}'" + ) @pytest.mark.asyncio @@ -419,9 +384,7 @@ async def test_async_pre_call_deployment_hook_prefers_top_level_model_info(): When model_info exists at top-level kwargs, async_pre_call_deployment_hook should use it without falling back to litellm_metadata. """ - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) managed_file_id = "managed-file-abc" top_level_model_id = "deployment-top" @@ -443,25 +406,25 @@ async def test_async_pre_call_deployment_hook_prefers_top_level_model_info(): }, } - result = await proxy_managed_files.async_pre_call_deployment_hook( - kwargs=kwargs, call_type=CallTypes.acreate_batch - ) + result = await proxy_managed_files.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.acreate_batch) - assert ( - result["input_file_id"] == top_level_provider_file - ), "Should prefer top-level model_info over litellm_metadata" + assert result["input_file_id"] == top_level_provider_file, ( + "Should prefer top-level model_info over litellm_metadata" + ) @pytest.mark.asyncio async def test_async_pre_call_deployment_hook_reads_model_info_that_forwards_its_fields(): - proxy_managed_files: Final = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=MagicMock()) + proxy_managed_files: Final = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) kwargs: Final = { "input_file_id": "managed-file-abc", "model_file_id_mapping": {"managed-file-abc": {"deployment-xyz": "file-provider-123"}}, "model_info": AttributeForwarder({"id": "deployment-xyz"}), } - result: Final = await proxy_managed_files.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.acreate_batch) + result: Final = await proxy_managed_files.async_pre_call_deployment_hook( + kwargs=kwargs, call_type=CallTypes.acreate_batch + ) assert result["input_file_id"] == "file-provider-123" @@ -472,9 +435,7 @@ async def test_async_pre_call_deployment_hook_no_model_info_leaves_file_id_uncha When model_info is absent from both top-level and litellm_metadata, the managed file ID should remain unchanged. """ - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) managed_file_id = "managed-file-abc" @@ -485,13 +446,11 @@ async def test_async_pre_call_deployment_hook_no_model_info_leaves_file_id_uncha }, } - result = await proxy_managed_files.async_pre_call_deployment_hook( - kwargs=kwargs, call_type=CallTypes.acreate_batch - ) + result = await proxy_managed_files.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.acreate_batch) - assert ( - result["input_file_id"] == managed_file_id - ), "File ID should remain unchanged when model_info is not available" + assert result["input_file_id"] == managed_file_id, ( + "File ID should remain unchanged when model_info is not available" + ) # def test_list_managed_files(): @@ -624,9 +583,7 @@ async def test_async_post_call_success_hook_for_unified_finetuning_job(): "unified_file_id": unified_file_id, "model_id": "gpt-3.5-turbo-0613", } - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=AsyncMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=AsyncMock()) data = { "user_api_key_dict": {"parent_otel_span": MagicMock()}, } @@ -649,13 +606,9 @@ async def test_async_pre_call_hook_for_unified_finetuning_job(): return_value = MagicMock() return_value.created_by = "123" prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) data = { - "user_api_key_dict": UserAPIKeyAuth( - user_id="123", parent_otel_span=MagicMock() - ), + "user_api_key_dict": UserAPIKeyAuth(user_id="123", parent_otel_span=MagicMock()), "data": { "fine_tuning_job_id": "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDo0OTIxODU4MWY3OGViZTllZjE4NDE0ZmE0ZjdmYjlmYTc0YzA5NWVkMTEyY2E4NDBkZDU2ZGZmZTliZDMwZGQxO2dlbmVyaWNfcmVzcG9uc2VfaWQ6ZnRqb2ItalRCeXM3YlZzYnlaRE93TDlHbHBZcVhS", }, @@ -668,9 +621,7 @@ async def test_async_pre_call_hook_for_unified_finetuning_job(): @pytest.mark.asyncio -@pytest.mark.parametrize( - "call_type", ["afile_content", "afile_delete", "afile_retrieve"] -) +@pytest.mark.parametrize("call_type", ["afile_content", "afile_delete", "afile_retrieve"]) async def test_can_user_call_unified_file_id(call_type): """ Test that on file retrieve, delete, and content we check if the user has access to the file @@ -681,16 +632,12 @@ async def test_can_user_call_unified_file_id(call_type): return_value = MagicMock() return_value.created_by = "123" prisma_client.db.litellm_managedfiletable.find_first.return_value = return_value - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxmMTNlNDAzZS01YWM3LTRhZjktOGQzNS0wNDgwZDMxOTgyYTg7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00by1taW5pLW9wZW5haTtsbG1fb3V0cHV0X2ZpbGVfaWQsZmlsZS1Ib3UxZDFXc3c1SDNKcjFMYllpZDJiO2xsbV9vdXRwdXRfZmlsZV9tb2RlbF9pZCxmODBiNWU2NzQ1NzdkNjkyMjM4YmVhNTIxZDdiMGI5ZGYyY2FmMTEwMTU2YmU5YzBjM2NjMmNkNTBjOTM1ZDI0" with pytest.raises(HTTPException): await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="456", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="456", parent_otel_span=MagicMock()), cache=MagicMock(), data={"file_id": unified_file_id}, call_type=call_type, @@ -708,9 +655,7 @@ async def test_router_acreate_batch_only_selects_from_file_id_mapping(monkeypatc return_value = MagicMock() return_value.created_by = "123" prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) monkeypatch.setattr( litellm, @@ -737,9 +682,7 @@ async def test_router_acreate_batch_only_selects_from_file_id_mapping(monkeypatc model_file_id_mapping = {file_id: {"5678": "file-LLCUFB52uTMa9hNYjtew9V"}} - with patch.object( - litellm, "acreate_batch", return_value=AsyncMock() - ) as mock_acreate_batch: + with patch.object(litellm, "acreate_batch", return_value=AsyncMock()) as mock_acreate_batch: for _ in range(1000): await router.acreate_batch( model="gpt-5-mini", @@ -753,9 +696,9 @@ async def test_router_acreate_batch_only_selects_from_file_id_mapping(monkeypatc @pytest.mark.asyncio async def test_output_file_id_for_batch_retrieve(): - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -824,10 +767,7 @@ async def test_output_file_id_for_batch_retrieve(): stored_file_object = managed_file_table.rows[output_file_id].file_object assert stored_file_object is not None assert stored_file_object.id == output_file_id - assert ( - managed_file_table.upsert_calls[0][1]["create"]["file_object"] - == stored_file_object.model_dump_json() - ) + assert managed_file_table.upsert_calls[0][1]["create"]["file_object"] == stored_file_object.model_dump_json() @pytest.mark.asyncio @@ -837,9 +777,9 @@ async def test_output_file_id_preserves_target_model_names_when_model_name_missi (e.g. Vertex batch retrieve), unified output_file_id should still include target_model_names from the managed input file ID. """ - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -865,11 +805,7 @@ async def test_output_file_id_preserves_target_model_names_when_model_name_missi "llm_output_file_id,file-input-1;" "llm_output_file_model_id,model-id-1" ) - managed_input_file_id = ( - base64.urlsafe_b64encode(managed_input_file_payload.encode()) - .decode() - .rstrip("=") - ) + managed_input_file_id = base64.urlsafe_b64encode(managed_input_file_payload.encode()).decode().rstrip("=") batch._hidden_params = { "model_id": "model-id-1", @@ -904,9 +840,7 @@ async def test_output_file_id_preserves_target_model_names_when_model_name_missi response=batch, ) - decoded_output_file_id = is_base64_encoded_unified_file_id( - cast(LiteLLMBatch, response).output_file_id - ) + decoded_output_file_id = is_base64_encoded_unified_file_id(cast(LiteLLMBatch, response).output_file_id) assert decoded_output_file_id assert "target_model_names,gemini-2.5-pro" in cast(str, decoded_output_file_id) mock_retrieve.assert_awaited_once() @@ -914,10 +848,7 @@ async def test_output_file_id_preserves_target_model_names_when_model_name_missi stored_file_object = managed_file_table.rows[output_file_id].file_object assert stored_file_object is not None assert stored_file_object.id == output_file_id - assert ( - managed_file_table.upsert_calls[0][1]["create"]["file_object"] - == stored_file_object.model_dump_json() - ) + assert managed_file_table.upsert_calls[0][1]["create"]["file_object"] == stored_file_object.model_dump_json() @pytest.mark.asyncio @@ -925,9 +856,9 @@ async def test_error_file_id_for_failed_batch(): """ Test that the error_file_id is properly managed when a batch fails """ - import litellm.proxy.proxy_server as proxy_server_module from openai.types.batch import BatchRequestCounts + import litellm.proxy.proxy_server as proxy_server_module from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -991,7 +922,6 @@ async def test_error_file_id_for_failed_batch(): return_value=error_file_object, ) as mock_retrieve, ): - user_api_key_dict = UserAPIKeyAuth(user_id="test-user-123") response = await proxy_managed_files.async_post_call_success_hook( @@ -1010,10 +940,7 @@ async def test_error_file_id_for_failed_batch(): stored_file_object = managed_file_table.rows[error_file_id].file_object assert stored_file_object is not None assert stored_file_object.id == error_file_id - assert ( - managed_file_table.upsert_calls[0][1]["create"]["file_object"] - == stored_file_object.model_dump_json() - ) + assert managed_file_table.upsert_calls[0][1]["create"]["file_object"] == stored_file_object.model_dump_json() @pytest.mark.asyncio @@ -1048,17 +975,13 @@ async def test_async_post_call_success_hook_twice_assert_no_unique_violation(): "unified_batch_id": "litellm_proxy;model_id:12345679;llm_batch_id:batch_685c5e5d63988190b85bdb2147ba131d", } - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) # first retrieve batch tasks = [] first_create_task = asyncio.create_task with patch("asyncio.create_task") as mock_create_task: - mock_create_task.side_effect = ( - lambda coro: tasks.append(first_create_task(coro)) or tasks[-1] - ) + mock_create_task.side_effect = lambda coro: tasks.append(first_create_task(coro)) or tasks[-1] response = await proxy_managed_files.async_post_call_success_hook( data={}, @@ -1080,9 +1003,7 @@ async def test_async_post_call_success_hook_twice_assert_no_unique_violation(): tasks = [] second_create_task = asyncio.create_task with patch("asyncio.create_task") as mock_create_task: - mock_create_task.side_effect = ( - lambda coro: tasks.append(second_create_task(coro)) or tasks[-1] - ) + mock_create_task.side_effect = lambda coro: tasks.append(second_create_task(coro)) or tasks[-1] await proxy_managed_files.async_post_call_success_hook( data={}, @@ -1134,10 +1055,7 @@ def test_update_responses_input_with_unified_file_id(): assert updated_input[0]["content"][0]["type"] == "input_file" assert updated_input[0]["content"][0]["file_id"] == "file-ECBPW7ML9g7XHdwGgUPZaM" assert updated_input[0]["content"][1]["type"] == "input_text" - assert ( - updated_input[0]["content"][1]["text"] - == "What is the first dragon in the book?" - ) + assert updated_input[0]["content"][1]["text"] == "What is the first dragon in the book?" def test_update_responses_input_with_regular_file_id(): @@ -1424,9 +1342,7 @@ def test_get_file_ids_from_responses_tools(): Test that get_file_ids_from_responses_tools correctly extracts file IDs from the tools parameter. """ - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) tools = [ { @@ -1447,9 +1363,7 @@ def test_get_file_ids_from_responses_tools_multiple_tools(): """ Test that get_file_ids_from_responses_tools handles multiple tools. """ - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) tools = [ { @@ -1481,9 +1395,7 @@ def test_get_file_ids_from_responses_tools_empty(): """ Test that get_file_ids_from_responses_tools handles empty or None tools. """ - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) # Test with None file_ids = proxy_managed_files.get_file_ids_from_responses_tools(None) @@ -1534,9 +1446,7 @@ async def test_check_file_ids_access_with_unified_file_ids(): ) # Verify can_user_call_unified_file_id was called for the unified file ID - proxy_managed_files.can_user_call_unified_file_id.assert_called_once_with( - unified_file_id, user_api_key_dict - ) + proxy_managed_files.can_user_call_unified_file_id.assert_called_once_with(unified_file_id, user_api_key_dict) @pytest.mark.asyncio @@ -1745,9 +1655,7 @@ async def test_store_unified_file_id_with_none_file_object(): from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - prisma_client.db.litellm_managedfiletable.upsert = AsyncMock( - return_value=MagicMock() - ) + prisma_client.db.litellm_managedfiletable.upsert = AsyncMock(return_value=MagicMock()) internal_usage_cache = MagicMock() internal_usage_cache.async_set_cache = AsyncMock() @@ -1780,11 +1688,8 @@ async def test_store_unified_file_id_updates_file_metadata_on_existing_row(): from litellm.types.llms.openai import OpenAIFileObject prisma_client = AsyncMock() - prisma_client.db.litellm_managedfiletable.upsert = AsyncMock( - return_value=MagicMock() - ) - internal_usage_cache = MagicMock() - internal_usage_cache.async_set_cache = AsyncMock() + prisma_client.db.litellm_managedfiletable.upsert = AsyncMock(return_value=MagicMock()) + internal_usage_cache = DualCache() proxy_managed_files = PROXY_LiteLLMManagedFiles( internal_usage_cache=internal_usage_cache, @@ -1822,12 +1727,8 @@ async def test_store_unified_file_id_updates_file_metadata_on_existing_row(): user_api_key_dict=user_api_key_dict, ) - first_update = prisma_client.db.litellm_managedfiletable.upsert.await_args_list[ - 0 - ].kwargs["data"]["update"] - second_update = prisma_client.db.litellm_managedfiletable.upsert.await_args_list[ - 1 - ].kwargs["data"]["update"] + first_update = prisma_client.db.litellm_managedfiletable.upsert.await_args_list[0].kwargs["data"]["update"] + second_update = prisma_client.db.litellm_managedfiletable.upsert.await_args_list[1].kwargs["data"]["update"] assert "file_object" not in first_update assert second_update["file_object"] == file_object.model_dump_json() assert second_update["storage_backend"] == "s3" @@ -1854,9 +1755,7 @@ async def test_store_batch_output_file_skips_existing_file_object(): flat_model_file_ids=["provider-output"], created_by="user-123", ) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - existing_file_row - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(existing_file_row) with patch("litellm.afile_retrieve", new_callable=AsyncMock) as retrieve: await proxy_managed_files.store_batch_output_file( @@ -1868,9 +1767,7 @@ async def test_store_batch_output_file_skips_existing_file_object(): ) retrieve.assert_not_called() - assert managed_file_table.find_first_calls == [ - {"unified_file_id": "unified-output"} - ] + assert managed_file_table.find_first_calls == [{"unified_file_id": "unified-output"}] assert managed_file_table.upsert_calls == [] @@ -1895,9 +1792,7 @@ async def test_store_batch_output_file_stores_provider_object_with_unified_id(): async def record_sleep(delay: float) -> None: sleep_calls.append(delay) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - sleep=record_sleep - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(sleep=record_sleep) with ( patch.object(proxy_server_module, "llm_router", router), @@ -1926,13 +1821,8 @@ async def test_store_batch_output_file_stores_provider_object_with_unified_id(): assert stored_object is not None assert stored_object.id == "unified-output" assert stored_object.bytes == 123 - assert ( - managed_file_table.upsert_calls[0][1]["create"]["file_object"] - == stored_object.model_dump_json() - ) - assert managed_file_table.rows["unified-output"].model_mappings == { - "model-123": "provider-output" - } + assert managed_file_table.upsert_calls[0][1]["create"]["file_object"] == stored_object.model_dump_json() + assert managed_file_table.rows["unified-output"].model_mappings == {"model-123": "provider-output"} @pytest.mark.asyncio @@ -1953,9 +1843,7 @@ async def test_store_batch_output_file_retries_model_name_provider_fetch(): async def record_sleep(delay: float) -> None: sleep_calls.append(delay) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - sleep=record_sleep - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(sleep=record_sleep) with patch( "litellm.afile_retrieve", new_callable=AsyncMock, @@ -1998,9 +1886,7 @@ async def test_store_batch_output_file_marks_model_name_fallback_after_four_tran async def record_sleep(delay: float) -> None: sleep_calls.append(delay) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - sleep=record_sleep - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(sleep=record_sleep) with patch( "litellm.afile_retrieve", new_callable=AsyncMock, @@ -2063,13 +1949,8 @@ async def test_store_batch_output_file_falls_back_when_provider_retrieve_raises( assert stored_object.filename == "output.jsonl" assert stored_object.bytes == 321 assert stored_object.purpose == "batch_output" - assert ( - managed_file_table.upsert_calls[0][1]["create"]["file_object"] - == stored_object.model_dump_json() - ) - assert managed_file_table.rows["unified-output"].model_mappings == { - "model-123": "s3://bucket/output.jsonl" - } + assert managed_file_table.upsert_calls[0][1]["create"]["file_object"] == stored_object.model_dump_json() + assert managed_file_table.rows["unified-output"].model_mappings == {"model-123": "s3://bucket/output.jsonl"} @pytest.mark.asyncio @@ -2163,10 +2044,7 @@ async def test_store_batch_output_file_falls_back_without_model_id(): assert stored_object.filename == "error.jsonl" assert stored_object.bytes == 0 assert stored_object.purpose == "batch_output" - assert ( - managed_file_table.upsert_calls[0][1]["create"]["file_object"] - == stored_object.model_dump_json() - ) + assert managed_file_table.upsert_calls[0][1]["create"]["file_object"] == stored_object.model_dump_json() assert managed_file_table.rows["unified-error"].model_mappings == {} assert stored_object.litellm_details_fallback is None assert "litellm_details_fallback" not in stored_object.model_dump() @@ -2182,9 +2060,7 @@ async def test_listed_batch_saves_basic_output_without_provider_fetch(): patch.object(proxy_server_module, "llm_router", None), patch("litellm.afile_retrieve", new_callable=AsyncMock) as retrieve, ): - resolved: Final = await _resolve_batch_for_output_listing( - proxy_managed_files, response - ) + resolved: Final = await _resolve_batch_for_output_listing(proxy_managed_files, response) assert resolved is response retrieve.assert_not_called() @@ -2206,18 +2082,14 @@ async def test_listed_batch_leaves_existing_marked_output_untouched(): model_id="model-123", model_name="bedrock/model-x", ) - row: Final = _marked_fallback_file_row().model_copy( - update={"unified_file_id": unified_file_id} - ) + row: Final = _marked_fallback_file_row().model_copy(update={"unified_file_id": unified_file_id}) proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(row) response: Final = _batch_response_for_output_listing() with ( patch.object(proxy_server_module, "llm_router", None), patch("litellm.afile_retrieve", new_callable=AsyncMock) as retrieve, ): - resolved: Final = await _resolve_batch_for_output_listing( - proxy_managed_files, response - ) + resolved: Final = await _resolve_batch_for_output_listing(proxy_managed_files, response) await proxy_managed_files.store_batch_output_file( unified_file_id=unified_file_id, provider_file_id="provider-output", @@ -2325,9 +2197,7 @@ async def test_batch_file_detail_retry_classification( async def record_sleep(delay: float) -> None: sleep_calls.append(delay) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - sleep=record_sleep - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(sleep=record_sleep) router: Final = MagicMock() router.registry.return_value.credentials.return_value = {"api_key": "key"} provider_file_object: Final = OpenAIFileObject( @@ -2339,11 +2209,7 @@ async def test_batch_file_detail_retry_classification( purpose="batch_output", status="processed", ) - retrieve_side_effect: Final = ( - [error_factory(), provider_file_object] - if expected_attempts == 2 - else error_factory() - ) + retrieve_side_effect: Final = [error_factory(), provider_file_object] if expected_attempts == 2 else error_factory() with ( patch.object(proxy_server_module, "llm_router", router), patch( @@ -2361,9 +2227,7 @@ async def test_batch_file_detail_retry_classification( ) assert retrieve.await_count == expected_attempts - assert [call.kwargs["max_retries"] for call in retrieve.call_args_list] == [ - 0 - ] * expected_attempts + assert [call.kwargs["max_retries"] for call in retrieve.call_args_list] == [0] * expected_attempts assert sleep_calls == ([0.5] if expected_attempts == 2 else []) stored_object: Final = managed_file_table.rows["unified-output"].file_object assert stored_object is not None @@ -2391,9 +2255,7 @@ async def test_store_batch_output_file_ignores_unmarked_row_with_provider_route( flat_model_file_ids=["provider-output"], created_by="user-123", ) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - existing_file_row - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(existing_file_row) router: Final = MagicMock() router.registry.return_value.credentials.return_value = {"api_key": "key"} with ( @@ -2665,9 +2527,7 @@ class _RefreshLandsBeforeTheFirstWriteTable(_InMemoryManagedFileTable): async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: if self.pending_refreshes: refreshed_row: Final = self.rows["unified-output"] - self.rows["unified-output"] = refreshed_row.model_copy( - update={"file_object": self.pending_refreshes.pop()} - ) + self.rows["unified-output"] = refreshed_row.model_copy(update={"file_object": self.pending_refreshes.pop()}) return await super().update_many(where, data) @@ -2936,9 +2796,7 @@ _NON_XAI_BATCH_OUTPUTS: Final = pytest.mark.parametrize( ) -def _non_xai_batch_output_row( - provider_file_id: str, *, size_bytes: int, fallback: bool -) -> LiteLLM_ManagedFileTable: +def _non_xai_batch_output_row(provider_file_id: str, *, size_bytes: int, fallback: bool) -> LiteLLM_ManagedFileTable: return _batch_output_row("xai", size_bytes=size_bytes, fallback=fallback).model_copy( update={"model_mappings": {"model-123": provider_file_id}, "flat_model_file_ids": [provider_file_id]} ) @@ -3117,9 +2975,7 @@ async def test_store_batch_output_file_marks_fallback_after_four_transient_failu async def record_sleep(delay: float) -> None: sleep_calls.append(delay) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - sleep=record_sleep - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(sleep=record_sleep) with ( patch.object(proxy_server_module, "llm_router", router), patch( @@ -3177,9 +3033,7 @@ async def test_store_batch_output_file_skips_lookup_for_fallback_written_moments assert stored_object is not None assert (stored_object.bytes, stored_object.litellm_details_fallback) == (836, True) assert managed_file_table.upsert_calls == [] - assert managed_file_table.update_many_calls[0][0] == { - "unified_file_id": "unified-output" - } + assert managed_file_table.update_many_calls[0][0] == {"unified_file_id": "unified-output"} assert set(managed_file_table.update_many_calls[0][1]) == {"file_object"} @@ -3195,9 +3049,7 @@ async def test_store_batch_output_file_does_not_retry_not_found(): async def record_sleep(delay: float) -> None: sleep_calls.append(delay) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - sleep=record_sleep - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(sleep=record_sleep) with ( patch.object(proxy_server_module, "llm_router", router), patch( @@ -3278,9 +3130,7 @@ async def test_store_batch_output_file_does_not_recreate_deleted_row_after_refre purpose="batch_output", status="processed", ) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - _marked_fallback_file_row() - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(_marked_fallback_file_row()) router: Final = MagicMock() router.registry.return_value.credentials.return_value = {"api_key": "key"} @@ -3307,9 +3157,7 @@ async def test_store_batch_output_file_does_not_recreate_deleted_row_after_refre assert "unified-output" not in managed_file_table.rows assert managed_file_table.upsert_calls == [] assert len(managed_file_table.update_many_calls) == 1 - saved_file_object_json: Final = cast( - str, managed_file_table.update_many_calls[0][1]["file_object"] - ) + saved_file_object_json: Final = cast(str, managed_file_table.update_many_calls[0][1]["file_object"]) saved_file_object: Final = json.loads(saved_file_object_json) assert saved_file_object["id"] == "unified-output" assert ( @@ -3328,9 +3176,7 @@ async def test_store_batch_output_file_keeps_marked_fallback_when_retryable_deta router = MagicMock() router.registry.return_value.credentials.return_value = {"api_key": "key"} - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - _marked_fallback_file_row() - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(_marked_fallback_file_row()) with ( patch.object(proxy_server_module, "llm_router", router), patch( @@ -3359,9 +3205,7 @@ async def test_store_batch_output_file_updates_only_size_for_failed_marked_fallb router = MagicMock() router.registry.return_value.credentials.return_value = {"api_key": "key"} - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - _marked_fallback_file_row() - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(_marked_fallback_file_row()) with ( patch.object(proxy_server_module, "llm_router", router), patch( @@ -3399,23 +3243,21 @@ async def test_afile_delete_returns_provider_response_when_stored_file_object_no prisma_client = AsyncMock() db_record = MagicMock() db_record.model_mappings = '{"model-123": "file-provider-xyz"}' - prisma_client.db.litellm_managedfiletable.find_first = AsyncMock( - return_value=db_record - ) + prisma_client.db.litellm_managedfiletable.find_first = AsyncMock(return_value=db_record) prisma_client.db.litellm_managedfiletable.delete = AsyncMock() - internal_usage_cache = MagicMock() - internal_usage_cache.async_get_cache = AsyncMock( - return_value={ + internal_usage_cache = DualCache() + await internal_usage_cache.async_set_cache( + key=unified_file_id, + value={ "unified_file_id": unified_file_id, "model_mappings": {"model-123": "file-provider-xyz"}, "flat_model_file_ids": ["file-provider-xyz"], "file_object": None, "created_by": "test-user", "updated_by": "test-user", - } + }, ) - internal_usage_cache.async_set_cache = AsyncMock() proxy_managed_files = PROXY_LiteLLMManagedFiles( internal_usage_cache=internal_usage_cache, @@ -3469,9 +3311,7 @@ async def test_afile_retrieve_fetches_from_provider_when_file_object_none(): async def record_sleep(delay: float) -> None: sleep_calls.append(delay) - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - stored_file, sleep=record_sleep - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(stored_file, sleep=record_sleep) # Mock the router and provider response provider_file_response = OpenAIFileObject( @@ -3531,7 +3371,7 @@ async def test_afile_retrieve_raises_error_when_no_router_and_file_object_none() unified_file_id = "test-unified-file-id" - with pytest.raises(Exception, match='LiteLLM Managed File object with id=test-unified-file-id') as exc_info: + with pytest.raises(Exception, match="LiteLLM Managed File object with id=test-unified-file-id") as exc_info: await proxy_managed_files.afile_retrieve( file_id=unified_file_id, litellm_parent_otel_span=None, @@ -3610,9 +3450,7 @@ async def test_afile_retrieve_refreshes_marked_fallback_and_preserves_ownership( assert updated_row.file_object == response assert updated_row.created_by == "user-123" assert updated_row.team_id == "team-123" - assert managed_file_table.update_many_calls[0][1] == { - "file_object": response.model_dump_json() - } + assert managed_file_table.update_many_calls[0][1] == {"file_object": response.model_dump_json()} assert managed_file_table.upsert_calls == [] cached_row = await proxy_managed_files.internal_usage_cache.async_get_cache( key="unified-output", @@ -3693,9 +3531,7 @@ async def test_afile_retrieve_uses_model_name_when_router_does_not_know_deployme assert response.bytes == 836 retrieve.assert_awaited_once() - router.registry.return_value.credentials.assert_called_once_with( - "bedrock/model-x", form="with_provider" - ) + router.registry.return_value.credentials.assert_called_once_with("bedrock/model-x", form="with_provider") assert retrieve.await_args.kwargs["custom_llm_provider"] == "bedrock" assert retrieve.await_args.kwargs["max_retries"] == 0 assert "_litellm_internal_model_credentials" not in retrieve.await_args.kwargs @@ -3717,13 +3553,9 @@ async def test_afile_retrieve_does_not_recreate_deleted_row_after_refresh(): router.registry.return_value.credentials.return_value = {"api_key": "key"} proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(row) - async def delete_row_then_return_file( - *, file_id: str, **_options: object - ) -> OpenAIFileObject: + async def delete_row_then_return_file(*, file_id: str, **_options: object) -> OpenAIFileObject: assert file_id == "provider-output" - deleted_row: Final = await managed_file_table.delete( - where={"unified_file_id": "unified-output"} - ) + deleted_row: Final = await managed_file_table.delete(where={"unified_file_id": "unified-output"}) assert deleted_row is not None return provider_object @@ -3782,10 +3614,7 @@ async def test_afile_retrieve_case3_includes_provider_error_text(): ) assert type(error.value) is Exception - assert str(error.value) == ( - "Failed to retrieve file unified-output from provider: " - "404: provider file is missing" - ) + assert str(error.value) == ("Failed to retrieve file unified-output from provider: 404: provider file is missing") @pytest.mark.asyncio @@ -3902,9 +3731,7 @@ async def test_afile_retrieve_case3_without_route_raises_accurate_error(): async def test_afile_retrieve_returns_marked_fallback_without_marker_when_refresh_fails(): router = MagicMock() router.registry.return_value.credentials.return_value = {"api_key": "key"} - proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma( - _marked_fallback_file_row() - ) + proxy_managed_files, managed_file_table = _managed_files_with_fake_prisma(_marked_fallback_file_row()) with patch( "litellm.afile_retrieve", @@ -3932,7 +3759,7 @@ async def test_afile_retrieve_raises_error_for_non_managed_file(): """ proxy_managed_files, _ = _managed_files_with_fake_prisma() - with pytest.raises(Exception, match='LiteLLM Managed File object with id=non-existent-file-id') as exc_info: + with pytest.raises(Exception, match="LiteLLM Managed File object with id=non-existent-file-id") as exc_info: await proxy_managed_files.afile_retrieve( file_id="non-existent-file-id", litellm_parent_otel_span=None, @@ -3943,7 +3770,6 @@ async def test_afile_retrieve_raises_error_for_non_managed_file(): @pytest.mark.asyncio async def test_list_batches_from_managed_objects_table(): - from openai.types.batch import BatchRequestCounts from litellm.proxy._types import UserAPIKeyAuth @@ -3984,9 +3810,7 @@ async def test_list_batches_from_managed_objects_table(): batch_record_2, ] - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) result = await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), @@ -4015,9 +3839,7 @@ async def test_list_batches_from_managed_objects_table_empty_list(): prisma_client = AsyncMock() prisma_client.db.litellm_managedobjecttable.find_many.return_value = [] - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) result = await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), @@ -4094,23 +3916,17 @@ async def test_list_batches_registers_and_returns_unified_output_file_ids(): prisma_client = AsyncMock() prisma_client.db.litellm_managedobjecttable.find_many.return_value = [ - _terminal_batch_record( - unified_batch_uid, raw_input_file_id, raw_output_file_id, raw_error_file_id - ) + _terminal_batch_record(unified_batch_uid, raw_input_file_id, raw_output_file_id, raw_error_file_id) ] input_file_row = MagicMock() input_file_row.unified_file_id = unified_input_file_id input_file_row.flat_model_file_ids = [raw_input_file_id] - prisma_client.db.litellm_managedfiletable.find_many = AsyncMock( - return_value=[input_file_row] - ) + prisma_client.db.litellm_managedfiletable.find_many = AsyncMock(return_value=[input_file_row]) prisma_client.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) result = await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="owner-user"), @@ -4138,9 +3954,7 @@ async def test_list_batches_registers_and_returns_unified_output_file_ids(): assert f"llm_output_file_id,{raw_error_file_id}" in decoded_error upsert_calls = prisma_client.db.litellm_managedfiletable.upsert.await_args_list - stored_raw_ids = { - c.kwargs["data"]["create"]["flat_model_file_ids"][0] for c in upsert_calls - } + stored_raw_ids = {c.kwargs["data"]["create"]["flat_model_file_ids"][0] for c in upsert_calls} assert stored_raw_ids == {raw_output_file_id, raw_error_file_id} for c in upsert_calls: assert c.kwargs["data"]["create"]["created_by"] == "owner-user" @@ -4151,9 +3965,7 @@ async def test_list_batches_registers_and_returns_unified_output_file_ids(): async def test_afile_list_returns_persisted_batch_output_file(): from litellm.proxy._types import UserAPIKeyAuth - proxy_managed_files, _ = _managed_files_with_fake_prisma( - _marked_fallback_file_row() - ) + proxy_managed_files, _ = _managed_files_with_fake_prisma(_marked_fallback_file_row()) result = await proxy_managed_files.afile_list( purpose="batch_output", @@ -4174,9 +3986,7 @@ async def test_afile_list_returns_persisted_batch_output_file(): async def test_get_user_created_file_ids_strips_fallback_marker(): from litellm.proxy._types import UserAPIKeyAuth - proxy_managed_files, _ = _managed_files_with_fake_prisma( - _marked_fallback_file_row() - ) + proxy_managed_files, _ = _managed_files_with_fake_prisma(_marked_fallback_file_row()) files = await proxy_managed_files.get_user_created_file_ids( UserAPIKeyAuth(user_id="user-123"), @@ -4224,14 +4034,10 @@ async def test_list_batches_resolves_existing_managed_rows_without_minting(): for raw_id, unified_id in zip(raw_output_file_ids, existing_unified_output_ids) ] - prisma_client.db.litellm_managedfiletable.find_many = AsyncMock( - return_value=existing_rows - ) + prisma_client.db.litellm_managedfiletable.find_many = AsyncMock(return_value=existing_rows) prisma_client.db.litellm_managedfiletable.find_first = AsyncMock() - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) result = await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="owner-user"), @@ -4254,19 +4060,14 @@ async def test_list_batches_caps_page_size_at_100(): prisma_client = AsyncMock() prisma_client.db.litellm_managedobjecttable.find_many.return_value = [] - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) result = await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="owner-user"), limit=100000, ) - assert ( - prisma_client.db.litellm_managedobjecttable.find_many.await_args.kwargs["take"] - == 101 - ) + assert prisma_client.db.litellm_managedobjecttable.find_many.await_args.kwargs["take"] == 101 assert result["data"] == [] @@ -4276,9 +4077,7 @@ async def test_list_batches_from_managed_objects_table_provider_filter_raises_ex prisma_client = AsyncMock() - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) # Filtering by provider should raise Exception with pytest.raises(Exception, match="Filtering by 'provider' is not supported when using managed") as exc_info: @@ -4288,9 +4087,7 @@ async def test_list_batches_from_managed_objects_table_provider_filter_raises_ex provider="openai", ) - assert str(exc_info.value) == ( - "Filtering by 'provider' is not supported when using managed batches." - ) + assert str(exc_info.value) == ("Filtering by 'provider' is not supported when using managed batches.") # Verify find_many was NOT called since exception is raised before database query prisma_client.db.litellm_managedobjecttable.find_many.assert_not_called() @@ -4302,9 +4099,7 @@ async def test_list_batches_from_managed_objects_table_target_model_name_filter_ prisma_client = AsyncMock() - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) # Filtering by provider should raise Exception with pytest.raises(Exception, match="Filtering by 'target_model_names' is not supported when") as exc_info: @@ -4314,9 +4109,7 @@ async def test_list_batches_from_managed_objects_table_target_model_name_filter_ target_model_names="gpt-5.5,gpt-3.5", ) - assert str(exc_info.value) == ( - "Filtering by 'target_model_names' is not supported when using managed batches." - ) + assert str(exc_info.value) == ("Filtering by 'target_model_names' is not supported when using managed batches.") # Verify find_many was NOT called since exception is raised before database query prisma_client.db.litellm_managedobjecttable.find_many.assert_not_called() @@ -4360,9 +4153,7 @@ async def test_list_batches_from_managed_objects_table_filters_by_created_by(): } ) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) # Query with user1's API key - should only return user1's batch prisma_client.db.litellm_managedobjecttable.find_many.return_value = [batch_user1] @@ -4410,9 +4201,7 @@ async def test_list_batches_pagination_uses_unified_object_id_cursor(): prisma_client.db.litellm_managedobjecttable.find_first.return_value = MagicMock() prisma_client.db.litellm_managedobjecttable.find_many.return_value = [] - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), @@ -4478,19 +4267,17 @@ async def test_list_batches_pagination_walks_all_pages_without_loops_or_gaps(): result = [r for r in result if r.id > id_filter["gt"]] order_keys = order if isinstance(order, list) else [order] for clause in reversed(order_keys): - (order_field, direction), = clause.items() - result.sort( - key=lambda r: getattr(r, order_field), reverse=(direction == "desc") - ) + ((order_field, direction),) = clause.items() + result.sort(key=lambda r: getattr(r, order_field), reverse=(direction == "desc")) if cursor is not None: - (cur_field, cur_val), = cursor.items() + ((cur_field, cur_val),) = cursor.items() idx = next( (i for i, r in enumerate(result) if getattr(r, cur_field) == cur_val), None, ) if idx is None: return [] - result = result[idx + skip:] + result = result[idx + skip :] return result[:take] async def fake_find_first(where): @@ -4500,24 +4287,16 @@ async def test_list_batches_pagination_walks_all_pages_without_loops_or_gaps(): ) prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( - side_effect=fake_find_many - ) - prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( - side_effect=fake_find_first - ) + prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(side_effect=fake_find_many) + prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(side_effect=fake_find_first) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) user = UserAPIKeyAuth(user_id="test-user") seen: list = [] after = None for _ in range(total + 5): - resp = await proxy_managed_files.list_user_batches( - user_api_key_dict=user, limit=3, after=after - ) + resp = await proxy_managed_files.list_user_batches(user_api_key_dict=user, limit=3, after=after) page_ids = [b.id for b in resp["data"]] seen.extend(page_ids) if not resp["has_more"]: @@ -4582,9 +4361,7 @@ async def test_list_batches_pagination_stable_when_created_at_ties(): result = list(rows) for clause in reversed(order_keys): field, direction = next(iter(clause.items())) - result.sort( - key=lambda r: getattr(r, field), reverse=(direction == "desc") - ) + result.sort(key=lambda r: getattr(r, field), reverse=(direction == "desc")) def order_key(r): return tuple(getattr(r, f) for f in fields) @@ -4611,7 +4388,7 @@ async def test_list_batches_pagination_stable_when_created_at_ties(): ) if idx is None: return [] - result = result[idx + skip:] + result = result[idx + skip :] return result[:take] async def fake_find_first(where): @@ -4621,24 +4398,16 @@ async def test_list_batches_pagination_stable_when_created_at_ties(): ) prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( - side_effect=fake_find_many - ) - prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( - side_effect=fake_find_first - ) + prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(side_effect=fake_find_many) + prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(side_effect=fake_find_first) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) user = UserAPIKeyAuth(user_id="test-user") seen: list = [] after = None for _ in range(total + 5): - resp = await proxy_managed_files.list_user_batches( - user_api_key_dict=user, limit=2, after=after - ) + resp = await proxy_managed_files.list_user_batches(user_api_key_dict=user, limit=2, after=after) page_ids = [b.id for b in resp["data"]] seen.extend(page_ids) if not resp["has_more"]: @@ -4678,18 +4447,16 @@ def _managed_batch_row(index, file_object=None): def _fake_managed_object_table(rows): async def find_many(where, take, order, cursor=None, skip=0): - result = sorted( - rows, key=lambda r: (r.created_at, r.unified_object_id), reverse=True - ) + result = sorted(rows, key=lambda r: (r.created_at, r.unified_object_id), reverse=True) if cursor is not None: - (cur_field, cur_val), = cursor.items() + ((cur_field, cur_val),) = cursor.items() idx = next( (i for i, r in enumerate(result) if getattr(r, cur_field) == cur_val), None, ) if idx is None: return [] - result = result[idx + skip:] + result = result[idx + skip :] return result[:take] async def find_first(where): @@ -4699,12 +4466,8 @@ def _fake_managed_object_table(rows): ) prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock( - side_effect=find_many - ) - prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( - side_effect=find_first - ) + prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(side_effect=find_many) + prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(side_effect=find_first) return prisma_client @@ -4712,9 +4475,7 @@ async def _walk_batch_pages(proxy_managed_files, user, limit, max_pages=20): pages = [] after = None for _ in range(max_pages): - resp = await proxy_managed_files.list_user_batches( - user_api_key_dict=user, limit=limit, after=after - ) + resp = await proxy_managed_files.list_user_batches(user_api_key_dict=user, limit=limit, after=after) pages.append(resp) if not resp["has_more"]: break @@ -4738,14 +4499,10 @@ async def test_list_batches_rejects_unknown_after_cursor(): from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( - return_value=None - ) + prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None) prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[]) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.list_user_batches( @@ -4778,9 +4535,7 @@ async def test_list_batches_treats_empty_after_as_no_cursor(): rows = [_managed_batch_row(i) for i in range(2)] prisma_client = _fake_managed_object_table(rows) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) page = await proxy_managed_files.list_user_batches( user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=2, after="" @@ -4815,14 +4570,10 @@ async def test_list_batches_rejects_after_cursor_owned_by_another_user(): return other_users_batch prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( - side_effect=find_first - ) + prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(side_effect=find_first) prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[]) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.list_user_batches( @@ -4848,13 +4599,9 @@ async def test_list_batches_has_more_false_on_exactly_full_final_page(): rows = [_managed_batch_row(i) for i in range(4)] prisma_client = _fake_managed_object_table(rows) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) - pages = await _walk_batch_pages( - proxy_managed_files, UserAPIKeyAuth(user_id="test-user"), limit=2 - ) + pages = await _walk_batch_pages(proxy_managed_files, UserAPIKeyAuth(user_id="test-user"), limit=2) seen = [batch.id for page in pages for batch in page["data"]] assert seen == [r.unified_object_id for r in reversed(rows)] @@ -4876,13 +4623,9 @@ async def test_list_batches_unparseable_row_does_not_truncate_pagination(): rows[2].file_object = "{ not valid json" prisma_client = _fake_managed_object_table(rows) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) - pages = await _walk_batch_pages( - proxy_managed_files, UserAPIKeyAuth(user_id="test-user"), limit=2 - ) + pages = await _walk_batch_pages(proxy_managed_files, UserAPIKeyAuth(user_id="test-user"), limit=2) seen = [batch.id for page in pages for batch in page["data"]] assert seen == [rows[3].unified_object_id, rows[1].unified_object_id, rows[0].unified_object_id] @@ -4905,13 +4648,9 @@ async def test_list_batches_fills_a_page_past_a_full_page_of_unparseable_rows(): corrupt_row.file_object = "{ not valid json" prisma_client = _fake_managed_object_table(rows) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) - pages = await _walk_batch_pages( - proxy_managed_files, UserAPIKeyAuth(user_id="test-user"), limit=1 - ) + pages = await _walk_batch_pages(proxy_managed_files, UserAPIKeyAuth(user_id="test-user"), limit=1) assert [[batch.id for batch in page["data"]] for page in pages] == [ [rows[4].unified_object_id], @@ -4931,25 +4670,17 @@ async def test_list_batches_bounds_the_queries_a_deep_unparseable_run_costs(): from litellm.proxy._types import UserAPIKeyAuth rows = [_managed_batch_row(0)] + [ - _managed_batch_row(index, file_object="{ not valid json") - for index in range(1, _DEEP_BATCH_SCAN_ROW_COUNT + 1) + _managed_batch_row(index, file_object="{ not valid json") for index in range(1, _DEEP_BATCH_SCAN_ROW_COUNT + 1) ] prisma_client = _fake_managed_object_table(rows) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) - page = await proxy_managed_files.list_user_batches( - user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=1 - ) + page = await proxy_managed_files.list_user_batches(user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=1) assert [batch.id for batch in page["data"]] == [rows[0].unified_object_id] assert page["has_more"] is False - assert ( - prisma_client.db.litellm_managedobjecttable.find_many.call_count - <= _DEEP_BATCH_SCAN_QUERY_BUDGET - ) + assert prisma_client.db.litellm_managedobjecttable.find_many.call_count <= _DEEP_BATCH_SCAN_QUERY_BUDGET @pytest.mark.asyncio @@ -4960,13 +4691,9 @@ async def test_list_batches_reads_one_chunk_when_the_first_one_fills_the_page(): rows = [_managed_batch_row(index) for index in range(_DEEP_BATCH_SCAN_ROW_COUNT)] prisma_client = _fake_managed_object_table(rows) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) - page = await proxy_managed_files.list_user_batches( - user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=2 - ) + page = await proxy_managed_files.list_user_batches(user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), limit=2) assert [batch.id for batch in page["data"]] == [ rows[-1].unified_object_id, @@ -5041,20 +4768,14 @@ async def test_user_b_cannot_retrieve_user_a_batch(): batch_record.created_by = "user_a_id" prisma_client.db.litellm_managedobjecttable.find_first.return_value = batch_record - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) # User B tries to retrieve User A's batch - unified_batch_id = ( - "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1tb2RlbDtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz" - ) + unified_batch_id = "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1tb2RlbDtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz" with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b_id", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b_id", parent_otel_span=MagicMock()), cache=MagicMock(), data={"batch_id": unified_batch_id}, call_type="aretrieve_batch", @@ -5078,20 +4799,14 @@ async def test_user_b_cannot_cancel_user_a_batch(): batch_record.created_by = "user_a_id" prisma_client.db.litellm_managedobjecttable.find_first.return_value = batch_record - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) # User B tries to cancel User A's batch - unified_batch_id = ( - "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1tb2RlbDtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz" - ) + unified_batch_id = "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1tb2RlbDtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz" with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b_id", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b_id", parent_otel_span=MagicMock()), cache=MagicMock(), data={"batch_id": unified_batch_id}, call_type="acancel_batch", @@ -5118,20 +4833,14 @@ async def test_user_a_can_retrieve_own_batch(): batch_record.created_by = "user_a_id" prisma_client.db.litellm_managedobjecttable.find_first.return_value = batch_record - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) # User A retrieves their own batch - unified_batch_id = ( - "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1tb2RlbDtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz" - ) + unified_batch_id = "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1tb2RlbDtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz" # Should not raise an exception result = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_a_id", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_a_id", parent_otel_span=MagicMock()), cache=MagicMock(), data={"batch_id": unified_batch_id}, call_type="aretrieve_batch", @@ -5156,20 +4865,14 @@ async def test_user_b_cannot_retrieve_user_a_file(): file_record.created_by = "user_a_id" prisma_client.db.litellm_managedfiletable.find_first.return_value = file_record - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) # User B tries to retrieve User A's file - unified_file_id = ( - "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9qc29uO3VuaWZpZWRfaWQsZmlsZS1hYmMxMjM" - ) + unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9qc29uO3VuaWZpZWRfaWQsZmlsZS1hYmMxMjM" with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b_id", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b_id", parent_otel_span=MagicMock()), cache=MagicMock(), data={"file_id": unified_file_id}, call_type="afile_retrieve", @@ -5193,20 +4896,14 @@ async def test_user_b_cannot_download_user_a_file_content(): file_record.created_by = "user_a_id" prisma_client.db.litellm_managedfiletable.find_first.return_value = file_record - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) # User B tries to download User A's file content - unified_file_id = ( - "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9qc29uO3VuaWZpZWRfaWQsZmlsZS1hYmMxMjM" - ) + unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9qc29uO3VuaWZpZWRfaWQsZmlsZS1hYmMxMjM" with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b_id", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b_id", parent_otel_span=MagicMock()), cache=MagicMock(), data={"file_id": unified_file_id}, call_type="afile_content", @@ -5230,20 +4927,14 @@ async def test_user_b_cannot_delete_user_a_file(): file_record.created_by = "user_a_id" prisma_client.db.litellm_managedfiletable.find_first.return_value = file_record - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) # User B tries to delete User A's file - unified_file_id = ( - "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9qc29uO3VuaWZpZWRfaWQsZmlsZS1hYmMxMjM" - ) + unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9qc29uO3VuaWZpZWRfaWQsZmlsZS1hYmMxMjM" with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b_id", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b_id", parent_otel_span=MagicMock()), cache=MagicMock(), data={"file_id": unified_file_id}, call_type="afile_delete", @@ -5280,20 +4971,14 @@ async def test_user_a_can_retrieve_own_file(): ) prisma_client.db.litellm_managedfiletable.find_first.return_value = file_record - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) # User A retrieves their own file - unified_file_id = ( - "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9qc29uO3VuaWZpZWRfaWQsZmlsZS1hYmMxMjM" - ) + unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9qc29uO3VuaWZpZWRfaWQsZmlsZS1hYmMxMjM" # Should not raise an exception result = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_a_id", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_a_id", parent_otel_span=MagicMock()), cache=MagicMock(), data={"file_id": unified_file_id}, call_type="afile_retrieve", @@ -5333,9 +5018,7 @@ async def test_list_batches_only_returns_user_own_batches(): # Mock database to only return User A's batches prisma_client.db.litellm_managedobjecttable.find_many.return_value = [batch_user_a] - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) # User A requests their batches result = await proxy_managed_files.list_user_batches( @@ -5372,19 +5055,13 @@ async def test_same_user_different_keys_can_access_batch(): batch_record.created_by = "user_a_id" prisma_client.db.litellm_managedobjecttable.find_first.return_value = batch_record - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) - unified_batch_id = ( - "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1tb2RlbDtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz" - ) + unified_batch_id = "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1tb2RlbDtsbG1fYmF0Y2hfaWQ6YmF0Y2hfYWJjMTIz" # First API key for User A retrieves the batch result1 = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_a_id", api_key="key-1", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_a_id", api_key="key-1", parent_otel_span=MagicMock()), cache=MagicMock(), data={"batch_id": unified_batch_id}, call_type="aretrieve_batch", @@ -5394,9 +5071,7 @@ async def test_same_user_different_keys_can_access_batch(): # Second API key for the same User A retrieves the batch result2 = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_a_id", api_key="key-2", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_a_id", api_key="key-2", parent_otel_span=MagicMock()), cache=MagicMock(), data={"batch_id": unified_batch_id}, call_type="aretrieve_batch", @@ -5407,12 +5082,8 @@ async def test_same_user_different_keys_can_access_batch(): assert result1["batch_id"] == result2["batch_id"] -MODEL_ENCODED_BATCH_ID = encode_file_id_with_model( - "batch_provider123", "gpt-4o-team-alias", id_type="batch" -) -MODEL_ENCODED_OUTPUT_FILE_ID = encode_file_id_with_model( - "file-output456", "gpt-4o-team-alias", id_type="file" -) +MODEL_ENCODED_BATCH_ID = encode_file_id_with_model("batch_provider123", "gpt-4o-team-alias", id_type="batch") +MODEL_ENCODED_OUTPUT_FILE_ID = encode_file_id_with_model("file-output456", "gpt-4o-team-alias", id_type="file") RAW_PROVIDER_BATCH_ID = "batch_provider123" RAW_PROVIDER_FILE_ID = "file-output456" @@ -5444,12 +5115,8 @@ def _batch_response(batch_id, output_file_id=None, is_create=False): @pytest.mark.asyncio @pytest.mark.parametrize("call_type", ["aretrieve_batch", "acancel_batch"]) -@pytest.mark.parametrize( - "batch_id", [MODEL_ENCODED_BATCH_ID, RAW_PROVIDER_BATCH_ID] -) -async def test_team_b_cannot_access_team_a_provider_format_batch( - call_type, batch_id -): +@pytest.mark.parametrize("batch_id", [MODEL_ENCODED_BATCH_ID, RAW_PROVIDER_BATCH_ID]) +async def test_team_b_cannot_access_team_a_provider_format_batch(call_type, batch_id): """ Cross-team retrieve/cancel of a model-encoded or raw provider batch id must 403 when an ownership row exists for another team. @@ -5461,18 +5128,14 @@ async def test_team_b_cannot_access_team_a_provider_format_batch( from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( - _owned_record(created_by="user_a", team_id="team_a") - ) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client + prisma_client.db.litellm_managedobjecttable.find_first.return_value = _owned_record( + created_by="user_a", team_id="team_a" ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b", team_id="team_b", parent_otel_span=MagicMock()), cache=MagicMock(), data={"batch_id": batch_id}, call_type=call_type, @@ -5502,17 +5165,13 @@ async def test_authorized_callers_can_access_provider_format_batch(caller_kwargs from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( - _owned_record(created_by="user_a", team_id="team_a") - ) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client + prisma_client.db.litellm_managedobjecttable.find_first.return_value = _owned_record( + created_by="user_a", team_id="team_a" ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) result = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - parent_otel_span=MagicMock(), **caller_kwargs - ), + user_api_key_dict=UserAPIKeyAuth(parent_otel_span=MagicMock(), **caller_kwargs), cache=MagicMock(), data={"batch_id": MODEL_ENCODED_BATCH_ID}, call_type="aretrieve_batch", @@ -5532,14 +5191,10 @@ async def test_provider_format_batch_without_ownership_row_stays_accessible(): prisma_client = AsyncMock() prisma_client.db.litellm_managedobjecttable.find_first.return_value = None - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) result = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b", team_id="team_b", parent_otel_span=MagicMock()), cache=MagicMock(), data={"batch_id": RAW_PROVIDER_BATCH_ID}, call_type="aretrieve_batch", @@ -5557,17 +5212,13 @@ async def test_fine_tuning_provider_format_id_not_enforced(): from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( - _owned_record(created_by="user_a", team_id="team_a") - ) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client + prisma_client.db.litellm_managedobjecttable.find_first.return_value = _owned_record( + created_by="user_a", team_id="team_a" ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) result = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b", team_id="team_b", parent_otel_span=MagicMock()), cache=MagicMock(), data={"fine_tuning_job_id": "ftjob-abc123"}, call_type="aretrieve_fine_tuning_job", @@ -5578,15 +5229,9 @@ async def test_fine_tuning_provider_format_id_not_enforced(): @pytest.mark.asyncio -@pytest.mark.parametrize( - "call_type", ["afile_content", "afile_retrieve", "afile_delete"] -) -@pytest.mark.parametrize( - "file_id", [MODEL_ENCODED_OUTPUT_FILE_ID, RAW_PROVIDER_FILE_ID] -) -async def test_team_b_cannot_access_team_a_provider_format_file( - call_type, file_id -): +@pytest.mark.parametrize("call_type", ["afile_content", "afile_retrieve", "afile_delete"]) +@pytest.mark.parametrize("file_id", [MODEL_ENCODED_OUTPUT_FILE_ID, RAW_PROVIDER_FILE_ID]) +async def test_team_b_cannot_access_team_a_provider_format_file(call_type, file_id): """ Cross-team content/retrieve/delete of a model-encoded or raw provider file id must 403 when an ownership row exists for another team. @@ -5594,18 +5239,14 @@ async def test_team_b_cannot_access_team_a_provider_format_file( from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - prisma_client.db.litellm_managedfiletable.find_first.return_value = ( - _owned_record(created_by="user_a", team_id="team_a") - ) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(), prisma_client=prisma_client + prisma_client.db.litellm_managedfiletable.find_first.return_value = _owned_record( + created_by="user_a", team_id="team_a" ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) with pytest.raises(HTTPException) as exc_info: await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b", team_id="team_b", parent_otel_span=MagicMock()), cache=MagicMock(), data={"file_id": file_id}, call_type=call_type, @@ -5622,17 +5263,13 @@ async def test_same_team_can_access_provider_format_file(): from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - prisma_client.db.litellm_managedfiletable.find_first.return_value = ( - _owned_record(created_by="user_a", team_id="team_a") - ) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(), prisma_client=prisma_client + prisma_client.db.litellm_managedfiletable.find_first.return_value = _owned_record( + created_by="user_a", team_id="team_a" ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) result = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="teammate_of_a", team_id="team_a", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="teammate_of_a", team_id="team_a", parent_otel_span=MagicMock()), cache=MagicMock(), data={"file_id": MODEL_ENCODED_OUTPUT_FILE_ID}, call_type="afile_content", @@ -5647,14 +5284,10 @@ async def test_provider_format_file_without_ownership_row_stays_accessible(): prisma_client = AsyncMock() prisma_client.db.litellm_managedfiletable.find_first.return_value = None - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(), prisma_client=prisma_client) result = await proxy_managed_files.async_pre_call_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b", team_id="team_b", parent_otel_span=MagicMock()), cache=MagicMock(), data={"file_id": RAW_PROVIDER_FILE_ID}, call_type="afile_content", @@ -5669,9 +5302,7 @@ async def test_post_call_batch_create_stores_ownership_row(batch_id): from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=prisma_client) await proxy_managed_files.async_post_call_success_hook( data={ @@ -5679,9 +5310,7 @@ async def test_post_call_batch_create_stores_ownership_row(batch_id): "endpoint": "/v1/chat/completions", "completion_window": "24h", }, - user_api_key_dict=UserAPIKeyAuth( - user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_a", team_id="team_a", parent_otel_span=MagicMock()), response=_batch_response(batch_id, is_create=True), ) @@ -5704,17 +5333,12 @@ async def test_post_call_batch_sync_does_not_claim_ownership(): prisma_client = AsyncMock() prisma_client.db.litellm_managedobjecttable.update_many.return_value = 0 - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(async_get_cache=AsyncMock(return_value=None), async_set_cache=AsyncMock()), - prisma_client=prisma_client, - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=prisma_client) prisma_client.db.litellm_managedfiletable.find_first.return_value = None await proxy_managed_files.async_post_call_success_hook( data={"batch_id": MODEL_ENCODED_BATCH_ID}, - user_api_key_dict=UserAPIKeyAuth( - user_id="user_b", team_id="team_b", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_b", team_id="team_b", parent_otel_span=MagicMock()), response=_batch_response(MODEL_ENCODED_BATCH_ID), ) @@ -5728,10 +5352,10 @@ async def test_post_call_batch_sync_updates_existing_row(): prisma_client = AsyncMock() prisma_client.db.litellm_managedobjecttable.update_many.return_value = 1 - prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( - _owned_record(created_by="user_a", team_id="team_a") + prisma_client.db.litellm_managedobjecttable.find_first.return_value = _owned_record( + created_by="user_a", team_id="team_a" ) - proxy_managed_files = PROXY_LiteLLMManagedFiles(MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=prisma_client) await proxy_managed_files.async_post_call_success_hook( data={"batch_id": MODEL_ENCODED_BATCH_ID}, @@ -5756,13 +5380,10 @@ async def test_post_call_batch_sync_stores_output_file_ownership_from_batch_row( prisma_client = AsyncMock() prisma_client.db.litellm_managedobjecttable.update_many.return_value = 1 - prisma_client.db.litellm_managedobjecttable.find_first.return_value = ( - _owned_record(created_by="user_a", team_id="team_a") - ) - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(async_get_cache=AsyncMock(return_value=None), async_set_cache=AsyncMock()), - prisma_client=prisma_client, + prisma_client.db.litellm_managedobjecttable.find_first.return_value = _owned_record( + created_by="user_a", team_id="team_a" ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=prisma_client) prisma_client.db.litellm_managedfiletable.find_first.return_value = None await proxy_managed_files.async_post_call_success_hook( @@ -5772,15 +5393,11 @@ async def test_post_call_batch_sync_stores_output_file_ownership_from_batch_row( user_role="proxy_admin", parent_otel_span=MagicMock(), ), - response=_batch_response( - MODEL_ENCODED_BATCH_ID, output_file_id=MODEL_ENCODED_OUTPUT_FILE_ID - ), + response=_batch_response(MODEL_ENCODED_BATCH_ID, output_file_id=MODEL_ENCODED_OUTPUT_FILE_ID), ) file_upsert = prisma_client.db.litellm_managedfiletable.upsert.await_args - assert file_upsert.kwargs["where"] == { - "unified_file_id": MODEL_ENCODED_OUTPUT_FILE_ID - } + assert file_upsert.kwargs["where"] == {"unified_file_id": MODEL_ENCODED_OUTPUT_FILE_ID} create_data = file_upsert.kwargs["data"]["create"] assert create_data["created_by"] == "user_a" assert create_data["team_id"] == "team_a" @@ -5792,9 +5409,7 @@ async def test_post_call_batch_create_does_not_store_output_file_ownership(): from litellm.proxy._types import UserAPIKeyAuth prisma_client = AsyncMock() - proxy_managed_files = PROXY_LiteLLMManagedFiles( - MagicMock(async_set_cache=AsyncMock()), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=prisma_client) await proxy_managed_files.async_post_call_success_hook( data={ @@ -5802,9 +5417,7 @@ async def test_post_call_batch_create_does_not_store_output_file_ownership(): "endpoint": "/v1/chat/completions", "completion_window": "24h", }, - user_api_key_dict=UserAPIKeyAuth( - user_id="user_a", team_id="team_a", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="user_a", team_id="team_a", parent_otel_span=MagicMock()), response=_batch_response( MODEL_ENCODED_BATCH_ID, output_file_id=MODEL_ENCODED_OUTPUT_FILE_ID, @@ -5842,15 +5455,11 @@ async def test_file_list_cursors_are_scoped_to_the_caller(): prisma_client = AsyncMock() prisma_client.db.litellm_managedfiletable.find_many.return_value = [] - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) response = await proxy_managed_files.async_post_call_success_hook( data={}, - user_api_key_dict=UserAPIKeyAuth( - user_id="other-user", team_id="other-team", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="other-user", team_id="other-team", parent_otel_span=MagicMock()), response=upstream_page, ) @@ -5899,15 +5508,11 @@ async def test_file_list_cursors_follow_the_owner_scoped_page(): } prisma_client = AsyncMock() prisma_client.db.litellm_managedfiletable.find_many.return_value = [managed_row] - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=prisma_client - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=prisma_client) response = await proxy_managed_files.async_post_call_success_hook( data={}, - user_api_key_dict=UserAPIKeyAuth( - user_id="mine-user", parent_otel_span=MagicMock() - ), + user_api_key_dict=UserAPIKeyAuth(user_id="mine-user", parent_otel_span=MagicMock()), response=upstream_page, ) @@ -5921,9 +5526,7 @@ async def test_file_list_cursors_follow_the_owner_scoped_page(): async def test_list_user_batches_provider_filter_rejected_with_400(): from litellm.proxy._types import ProxyException, UserAPIKeyAuth - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) with pytest.raises(ProxyException) as exc: await proxy_managed_files.list_user_batches( @@ -5941,9 +5544,7 @@ async def test_list_user_batches_provider_filter_rejected_with_400(): async def test_list_user_batches_target_model_names_filter_rejected_with_400(): from litellm.proxy._types import ProxyException, UserAPIKeyAuth - proxy_managed_files = PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() - ) + proxy_managed_files = PROXY_LiteLLMManagedFiles(InternalUsageCache(DualCache()), prisma_client=MagicMock()) with pytest.raises(ProxyException) as exc: await proxy_managed_files.list_user_batches( @@ -6262,7 +5863,9 @@ async def test_a_read_of_a_proxy_issued_response_id_is_marked_managed( monkeypatch.setenv("LITELLM_SALT_KEY", _MANAGED_READ_SALT_KEY) addressed_id: Final = _proxy_issued_response_id(encrypted) - result: Final = await PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=MagicMock()).async_pre_call_hook( + result: Final = await PROXY_LiteLLMManagedFiles( + InternalUsageCache(DualCache()), prisma_client=MagicMock() + ).async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(user_id="alice"), cache=MagicMock(), data={"response_id": addressed_id, "litellm_metadata": {"user_api_key_user_id": "alice"}}, @@ -6282,7 +5885,9 @@ async def test_a_read_of_a_raw_provider_response_id_is_not_marked_managed() -> N from litellm.constants import MANAGED_BACKGROUND_RESPONSE_METADATA_KEY from litellm.proxy._types import UserAPIKeyAuth - result: Final = await PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=MagicMock()).async_pre_call_hook( + result: Final = await PROXY_LiteLLMManagedFiles( + InternalUsageCache(DualCache()), prisma_client=MagicMock() + ).async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(user_id="alice"), cache=MagicMock(), data={"response_id": "resp_raw_provider_job", "metadata": {}}, @@ -6313,10 +5918,18 @@ async def test_only_a_streamed_background_create_is_marked_for_the_managed_bill( from litellm.constants import MANAGED_BACKGROUND_RESPONSE_METADATA_KEY from litellm.proxy._types import UserAPIKeyAuth - result: Final = await PROXY_LiteLLMManagedFiles(DualCache(), prisma_client=MagicMock()).async_pre_call_hook( + result: Final = await PROXY_LiteLLMManagedFiles( + InternalUsageCache(DualCache()), prisma_client=MagicMock() + ).async_pre_call_hook( user_api_key_dict=UserAPIKeyAuth(user_id="alice"), cache=MagicMock(), - data={"model": "gpt-5.6", "input": "hello", "metadata": {"run": "nightly"}, "litellm_metadata": {}, **create_fields}, + data={ + "model": "gpt-5.6", + "input": "hello", + "metadata": {"run": "nightly"}, + "litellm_metadata": {}, + **create_fields, + }, call_type="aresponses", ) diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index f2df14cc8a3..c0f6018469f 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -10,14 +10,15 @@ import base64 import json import logging from types import MappingProxyType +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch import pytest -from typing import Final, Optional -from unittest.mock import AsyncMock, MagicMock, patch from litellm.llms.base_llm.files.transformation import FileListModelScope from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.openai_files_endpoints.common_utils import BATCH_CREATE_HIDDEN_PARAM +from litellm.proxy.utils import InternalUsageCache from litellm.types.llms.openai import FileListPage, OpenAIFileObject from litellm.types.utils import LiteLLMBatch @@ -36,7 +37,7 @@ def _make_file_object(file_id: str = "file-output-abc") -> OpenAIFileObject: def _make_batch_response( batch_id: str = "batch-123", - output_file_id: Optional[str] = "file-output-abc", + output_file_id: str | None = "file-output-abc", status: str = "completed", model_id: str = "model-deploy-xyz", model_name: str = "azure/gpt-4", @@ -100,11 +101,9 @@ def _make_managed_file_row( unified_file_id: str, purpose: str = "batch_output", created_by: str = "test-user", - team_id: Optional[str] = None, + team_id: str | None = None, ) -> MagicMock: - file_object = _make_file_object(f"file-provider-{unified_file_id}").model_copy( - update={"purpose": purpose} - ) + file_object = _make_file_object(f"file-provider-{unified_file_id}").model_copy(update={"purpose": purpose}) return MagicMock( unified_file_id=unified_file_id, file_object=file_object.model_dump(), @@ -116,7 +115,7 @@ def _make_managed_file_row( def _make_unparseable_managed_file_row( unified_file_id: str, created_by: str = "test-user", - team_id: Optional[str] = None, + team_id: str | None = None, ) -> MagicMock: """A row whose stored blob cannot be parsed back into a file object.""" return MagicMock( @@ -156,16 +155,10 @@ class _FakeManagedFileTable: return next(iter(self._owned_rows(where)), None) async def find_many(self, where, take=None, order=None, cursor=None, skip=0): - self.find_many_calls.append( - {"where": where, "take": take, "order": order, "cursor": cursor, "skip": skip} - ) + self.find_many_calls.append({"where": where, "take": take, "order": order, "cursor": cursor, "skip": skip}) rows = self._owned_rows(where) if cursor is not None: - start = next( - index - for index, row in enumerate(rows) - if row.unified_file_id == cursor["unified_file_id"] - ) + start = next(index for index, row in enumerate(rows) if row.unified_file_id == cursor["unified_file_id"]) rows = rows[start + skip :] return rows if take is None else rows[:take] @@ -183,13 +176,14 @@ def _make_managed_files_instance(): PROXY_LiteLLMManagedFiles, ) - mock_cache = MagicMock() - mock_cache.async_get_cache = AsyncMock(return_value=None) + from litellm.caching.dual_cache import DualCache + + cache: Final = DualCache() mock_prisma = MagicMock() mock_prisma.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) instance = PROXY_LiteLLMManagedFiles( - internal_usage_cache=mock_cache, + internal_usage_cache=cache, prisma_client=mock_prisma, ) instance.store_unified_file_id = AsyncMock() @@ -222,9 +216,7 @@ async def test_should_pass_credentials_to_afile_retrieve(): } mock_router = MagicMock() - mock_router.registry.return_value.credentials = MagicMock( - return_value=mock_credentials - ) + mock_router.registry.return_value.credentials = MagicMock(return_value=mock_credentials) mock_afile_retrieve = AsyncMock(return_value=_make_file_object("file-output-abc")) @@ -242,14 +234,10 @@ async def test_should_pass_credentials_to_afile_retrieve(): call_kwargs = mock_afile_retrieve.call_args assert call_kwargs.kwargs.get("api_key") == "test-azure-key", ( - f"afile_retrieve must receive api_key from router credentials. " - f"Got kwargs: {call_kwargs.kwargs}" + f"afile_retrieve must receive api_key from router credentials. Got kwargs: {call_kwargs.kwargs}" ) - assert ( - call_kwargs.kwargs.get("api_base") == "https://my-azure.openai.azure.com/" - ), ( - f"afile_retrieve must receive api_base from router credentials. " - f"Got kwargs: {call_kwargs.kwargs}" + assert call_kwargs.kwargs.get("api_base") == "https://my-azure.openai.azure.com/", ( + f"afile_retrieve must receive api_base from router credentials. Got kwargs: {call_kwargs.kwargs}" ) @@ -266,9 +254,7 @@ async def test_get_user_created_file_ids_skips_rows_without_file_object(): ] ) - files = await managed_files.get_user_created_file_ids( - _make_user_api_key_dict(), ["file-output-abc"] - ) + files = await managed_files.get_user_created_file_ids(_make_user_api_key_dict(), ["file-output-abc"]) assert [file.id for file in files] == ["unified-id-1"] @@ -294,9 +280,7 @@ async def test_get_user_created_file_ids_remaps_stored_raw_provider_id_to_unifie ] ) - files = await managed_files.get_user_created_file_ids( - _make_user_api_key_dict(), ["file-raw-provider-123"] - ) + files = await managed_files.get_user_created_file_ids(_make_user_api_key_dict(), ["file-raw-provider-123"]) assert [file.id for file in files] == [unified_id] assert files[0].filename == raw_provider_object.filename @@ -304,9 +288,7 @@ async def test_get_user_created_file_ids_remaps_stored_raw_provider_id_to_unifie @pytest.mark.asyncio -async def test_provider_file_id_resolver_returns_owned_mappings_with_owner_scoped_filter() -> ( - None -): +async def test_provider_file_id_resolver_returns_owned_mappings_with_owner_scoped_filter() -> None: managed_files: Final = _make_managed_files_instance() managed_row: Final = MagicMock( unified_file_id="unified-file-id", @@ -315,16 +297,14 @@ async def test_provider_file_id_resolver_returns_owned_mappings_with_owner_scope find_many: Final = AsyncMock(return_value=[managed_row]) managed_files.prisma_client.db.litellm_managedfiletable.find_many = find_many - unified_file_ids: Final = ( - await managed_files.get_unified_file_ids_for_provider_file_ids( - provider_file_ids=( - "file-provider-1", - "file-provider-2", - "file-unmanaged-2", - "file-provider-1", - ), - user_api_key_dict=_make_team_member_api_key_dict(), - ) + unified_file_ids: Final = await managed_files.get_unified_file_ids_for_provider_file_ids( + provider_file_ids=( + "file-provider-1", + "file-provider-2", + "file-unmanaged-2", + "file-provider-1", + ), + user_api_key_dict=_make_team_member_api_key_dict(), ) assert unified_file_ids == { @@ -343,9 +323,7 @@ async def test_provider_file_id_resolver_returns_owned_mappings_with_owner_scope @pytest.mark.asyncio -async def test_provider_file_id_resolver_denies_unowned_callers_without_database_query() -> ( - None -): +async def test_provider_file_id_resolver_denies_unowned_callers_without_database_query() -> None: managed_files: Final = _make_managed_files_instance() find_many: Final = AsyncMock() managed_files.prisma_client.db.litellm_managedfiletable.find_many = find_many @@ -357,11 +335,9 @@ async def test_provider_file_id_resolver_denies_unowned_callers_without_database parent_otel_span=None, ) - unified_file_ids: Final = ( - await managed_files.get_unified_file_ids_for_provider_file_ids( - provider_file_ids=("file-provider-1",), - user_api_key_dict=no_owner, - ) + unified_file_ids: Final = await managed_files.get_unified_file_ids_for_provider_file_ids( + provider_file_ids=("file-provider-1",), + user_api_key_dict=no_owner, ) assert unified_file_ids == {} @@ -375,11 +351,9 @@ async def test_provider_file_id_resolver_skips_database_query_for_empty_input() find_many: Final = AsyncMock() managed_files.prisma_client.db.litellm_managedfiletable.find_many = find_many - unified_file_ids: Final = ( - await managed_files.get_unified_file_ids_for_provider_file_ids( - provider_file_ids=(), - user_api_key_dict=_make_user_api_key_dict(), - ) + unified_file_ids: Final = await managed_files.get_unified_file_ids_for_provider_file_ids( + provider_file_ids=(), + user_api_key_dict=_make_user_api_key_dict(), ) assert unified_file_ids == {} @@ -397,9 +371,9 @@ async def test_afile_list_returns_owner_scoped_managed_files(): unified_file_id="unified-file-id", ), MagicMock( - file_object=_make_file_object("file-other-purpose").model_copy( - update={"purpose": "batch"} - ).model_dump(), + file_object=_make_file_object("file-other-purpose") + .model_copy(update={"purpose": "batch"}) + .model_dump(), unified_file_id="unified-other-purpose", ), ] @@ -519,9 +493,7 @@ async def test_afile_list_returns_own_and_team_files_for_a_key_carrying_both_ids ) assert [file.id for file in response.data] == ["unified-mine", "unified-teammates"] - assert table.find_many_calls[0]["where"] == { - "OR": [{"created_by": "test-user"}, {"team_id": "test-team"}] - } + assert table.find_many_calls[0]["where"] == {"OR": [{"created_by": "test-user"}, {"team_id": "test-team"}]} @pytest.mark.asyncio @@ -848,9 +820,7 @@ async def test_afile_list_reads_one_chunk_when_the_first_one_fills_the_page(): @pytest.mark.asyncio async def test_afile_list_reports_no_more_pages_when_nothing_matches(): - managed_files, _ = _make_managed_files_over_rows( - [_make_managed_file_row(f"unified-{index}") for index in range(5)] - ) + managed_files, _ = _make_managed_files_over_rows([_make_managed_file_row(f"unified-{index}") for index in range(5)]) page = await managed_files.afile_list( purpose="batch", @@ -1029,9 +999,7 @@ async def test_get_user_created_file_ids_skips_unparseable_rows(): ] ) - files = await managed_files.get_user_created_file_ids( - _make_user_api_key_dict(), ["file-output-abc"] - ) + files = await managed_files.get_user_created_file_ids(_make_user_api_key_dict(), ["file-output-abc"]) assert [file.id for file in files] == ["unified-valid"] @@ -1096,9 +1064,7 @@ async def test_should_not_double_wrap_already_unified_output_file_id(): "custom_llm_provider": "openai", } mock_router = MagicMock() - mock_router.registry.return_value.credentials = MagicMock( - return_value=mock_credentials - ) + mock_router.registry.return_value.credentials = MagicMock(return_value=mock_credentials) mock_afile_retrieve = AsyncMock(return_value=_make_file_object(provider_file_id)) with ( @@ -1115,13 +1081,9 @@ async def test_should_not_double_wrap_already_unified_output_file_id(): mock_afile_retrieve.assert_called_once() assert mock_afile_retrieve.call_args.kwargs["file_id"] == provider_file_id managed_files.store_unified_file_id.assert_awaited_once() - assert managed_files.store_unified_file_id.await_args.kwargs["model_mappings"] == { - model_id: provider_file_id - } + assert managed_files.store_unified_file_id.await_args.kwargs["model_mappings"] == {model_id: provider_file_id} - decoded = base64.urlsafe_b64decode( - already_unified + "=" * (-len(already_unified) % 4) - ).decode() + decoded = base64.urlsafe_b64decode(already_unified + "=" * (-len(already_unified) % 4)).decode() assert decoded.count(f"llm_output_file_id,{provider_file_id}") == 1 @@ -1132,9 +1094,7 @@ async def test_should_skip_non_file_unified_id_on_output_file_id(): managed_files = _make_managed_files_instance() batch_unified = ( - base64.urlsafe_b64encode( - b"litellm_proxy;model_id:openai/openai/gpt-5.5-batch;llm_batch_id:batch_abc" - ) + base64.urlsafe_b64encode(b"litellm_proxy;model_id:openai/openai/gpt-5.5-batch;llm_batch_id:batch_abc") .decode() .rstrip("=") ) @@ -1172,9 +1132,7 @@ async def test_afile_content_passes_trusted_model_credentials_to_router(): unified_file_id = "unified-file-id" s3_uri = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" managed_files.get_unified_file_id = AsyncMock(return_value=None) - managed_files.get_model_file_id_mapping = AsyncMock( - return_value={unified_file_id: {"model-123": s3_uri}} - ) + managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}}) mock_router = MagicMock() mock_router.registry.return_value.credentials = MagicMock( @@ -1210,9 +1168,10 @@ def _managed_deletion_file_id(provider_file_id): def _managed_files_with_deletion_row(unified_file_id, provider_file_id, file_object): + from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles + from litellm.caching import DualCache from litellm.models.managed_files import LiteLLM_ManagedFileTable - from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles row = LiteLLM_ManagedFileTable( unified_file_id=unified_file_id, @@ -1225,7 +1184,7 @@ def _managed_files_with_deletion_row(unified_file_id, provider_file_id, file_obj delete=AsyncMock(), ) return PROXY_LiteLLMManagedFiles( - internal_usage_cache=DualCache(), + internal_usage_cache=InternalUsageCache(DualCache()), prisma_client=MagicMock(db=MagicMock(litellm_managedfiletable=table)), ), table @@ -1260,9 +1219,9 @@ async def test_afile_delete_bedrock_uses_deployment_bucket_and_signed_s3_delete( unified_file_id = _managed_deletion_file_id(s3_uri) managed_files, table = _managed_files_with_deletion_row(unified_file_id, s3_uri, None) with respx.mock: - route = respx.delete( - "https://s3.us-west-2.amazonaws.com/my-bucket/litellm-bedrock-files/input.jsonl" - ).mock(return_value=httpx.Response(204)) + route = respx.delete("https://s3.us-west-2.amazonaws.com/my-bucket/litellm-bedrock-files/input.jsonl").mock( + return_value=httpx.Response(204) + ) response = await managed_files.afile_delete( file_id=unified_file_id, litellm_parent_otel_span=None, @@ -1344,15 +1303,11 @@ async def test_afile_content_bedrock_unified_id_end_to_end(monkeypatch): unified_file_id = "unified-file-id" s3_uri = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" managed_files.get_unified_file_id = AsyncMock(return_value=None) - managed_files.get_model_file_id_mapping = AsyncMock( - return_value={unified_file_id: {"model-123": s3_uri}} - ) + managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}}) expected_url = "https://s3.us-west-2.amazonaws.com/my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" with respx.mock: - route = respx.get(expected_url).mock( - return_value=httpx.Response(200, content=b'{"recordId": "x"}') - ) + route = respx.get(expected_url).mock(return_value=httpx.Response(200, content=b'{"recordId": "x"}')) response = await managed_files.afile_content( file_id=unified_file_id, @@ -1361,9 +1316,7 @@ async def test_afile_content_bedrock_unified_id_end_to_end(monkeypatch): ) assert route.called - assert ( - route.calls[0].request.headers["Authorization"].startswith("AWS4-HMAC-SHA256") - ) + assert route.calls[0].request.headers["Authorization"].startswith("AWS4-HMAC-SHA256") assert response.content == b'{"recordId": "x"}' @@ -1375,15 +1328,13 @@ async def test_afile_content_error_reports_unified_id_not_provider_uri(): unified_file_id = "litellm_proxy_unified_id_abc" s3_uri = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" managed_files.get_unified_file_id = AsyncMock(return_value=None) - managed_files.get_model_file_id_mapping = AsyncMock( - return_value={unified_file_id: {"model-123": s3_uri}} - ) + managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}}) mock_router = MagicMock() mock_router.registry.return_value.credentials = MagicMock(return_value=None) mock_router.afile_content = AsyncMock(side_effect=Exception("deployment failed")) - with pytest.raises(Exception, match='LiteLLM Managed File object with') as exc_info: + with pytest.raises(Exception, match="LiteLLM Managed File object with") as exc_info: await managed_files.afile_content( file_id=unified_file_id, litellm_parent_otel_span=None, @@ -1402,17 +1353,15 @@ def _make_real_managed_files_instance(): PROXY_LiteLLMManagedFiles, ) - mock_cache = MagicMock() - mock_cache.async_get_cache = AsyncMock(return_value=None) - mock_cache.async_set_cache = AsyncMock() + from litellm.caching.dual_cache import DualCache + + mock_cache = DualCache() mock_prisma = MagicMock() mock_prisma.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) mock_prisma.db.litellm_managedfiletable.upsert = AsyncMock() mock_prisma.db.litellm_managedfiletable.create = AsyncMock( - side_effect=AssertionError( - "store_unified_file_id must upsert, not create, on the retrieve path" - ) + side_effect=AssertionError("store_unified_file_id must upsert, not create, on the retrieve path") ) return ( @@ -1454,9 +1403,7 @@ async def test_poll_refreshes_batch_state_without_claiming_the_row(): create-time snapshot forever -- while writing none of the attribution columns and never creating a row it would then own.""" managed_files, mock_prisma = _make_object_store_instance() - poller = UserAPIKeyAuth( - api_key="sk-the-poller", user_id="bob", team_id="team-bravo", parent_otel_span=None - ) + poller = UserAPIKeyAuth(api_key="sk-the-poller", user_id="bob", team_id="team-bravo", parent_otel_span=None) await managed_files.store_unified_object_id( unified_object_id="uoi-1", @@ -1490,9 +1437,7 @@ async def test_create_still_upserts_and_claims_attribution(): """The create is the one caller that can speak for the batch, so it keeps the upsert (creating the row when absent) and writes the attribution columns.""" managed_files, mock_prisma = _make_object_store_instance() - creator = UserAPIKeyAuth( - api_key="sk-the-creator", user_id="alice", team_id="team-alpha", parent_otel_span=None - ) + creator = UserAPIKeyAuth(api_key="sk-the-creator", user_id="alice", team_id="team-alpha", parent_otel_span=None) await managed_files.store_unified_object_id( unified_object_id="uoi-2", @@ -1672,8 +1617,7 @@ async def test_concurrent_first_registrations_converge_on_one_row(): def _b64_unified_input_file_id(target_model_names: str) -> str: unified_input_file_id = ( - "litellm_proxy:application/octet-stream;unified_id,input-uuid;" - f"target_model_names,{target_model_names}" + f"litellm_proxy:application/octet-stream;unified_id,input-uuid;target_model_names,{target_model_names}" ) return base64.urlsafe_b64encode(unified_input_file_id.encode()).decode().rstrip("=") @@ -1682,9 +1626,7 @@ def _b64_unified_input_file_id(target_model_names: str) -> str: async def test_hook_mint_prefers_input_file_target_model_names(): managed_files = _make_managed_files_instance() batch_response = _make_batch_response(model_name="model-a") - batch_response._hidden_params["unified_file_id"] = _b64_unified_input_file_id( - "model-a,model-b" - ) + batch_response._hidden_params["unified_file_id"] = _b64_unified_input_file_id("model-a,model-b") mock_router = MagicMock() mock_router.registry.return_value.credentials = MagicMock(return_value={}) @@ -1738,10 +1680,11 @@ async def test_hook_mint_falls_back_to_response_input_file_id_target_models(): @pytest.mark.asyncio async def test_cost_job_and_retrieve_paths_mint_identical_unified_output_file_ids(): + from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost + from litellm.proxy.openai_files_endpoints.common_utils import ( ensure_batch_response_managed_file_ids, ) - from litellm_enterprise.proxy.common_utils.check_batch_cost import CheckBatchCost managed_files, mock_prisma = _make_real_managed_files_instance() mock_prisma.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) @@ -2040,7 +1983,7 @@ async def test_afile_delete_storage_backed_row_deletes_stored_content_not_provid file_table = MagicMock(find_first=AsyncMock(return_value=row), delete=AsyncMock()) content_table = MagicMock(delete=AsyncMock()) managed_files = PROXY_LiteLLMManagedFiles( - internal_usage_cache=DualCache(), + internal_usage_cache=InternalUsageCache(DualCache()), prisma_client=MagicMock( db=MagicMock(litellm_managedfiletable=file_table, litellm_managedfilecontenttable=content_table) ), @@ -2084,7 +2027,7 @@ async def test_afile_content_storage_backed_row_returns_stored_bytes_not_provide file_table = MagicMock(find_first=AsyncMock(return_value=row)) content_table = MagicMock(find_unique=AsyncMock(return_value=MagicMock(content=Base64.encode(stored_bytes)))) managed_files = PROXY_LiteLLMManagedFiles( - internal_usage_cache=DualCache(), + internal_usage_cache=InternalUsageCache(DualCache()), prisma_client=MagicMock( db=MagicMock(litellm_managedfiletable=file_table, litellm_managedfilecontenttable=content_table) ), @@ -2142,7 +2085,7 @@ async def test_store_unified_file_id_caches_the_storage_location_the_db_row_gets file_table = MagicMock(upsert=AsyncMock(), find_first=AsyncMock(side_effect=AssertionError("cache miss"))) managed_files = PROXY_LiteLLMManagedFiles( - internal_usage_cache=DualCache(), + internal_usage_cache=InternalUsageCache(DualCache()), prisma_client=MagicMock(db=MagicMock(litellm_managedfiletable=file_table)), ) stored = _make_file_object("file-kept").model_copy(update={"purpose": "batch"}) diff --git a/tests/unit/redis/stores/test_managed_files.py b/tests/unit/redis/stores/test_managed_files.py new file mode 100644 index 00000000000..516ad0745c5 --- /dev/null +++ b/tests/unit/redis/stores/test_managed_files.py @@ -0,0 +1,264 @@ +import json +import sys +import time +from collections.abc import Mapping +from datetime import timedelta +from typing import Final, cast +from unittest.mock import patch + +import fakeredis +import pytest +from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles +from pydantic import ValidationError +from redis.asyncio import Redis +from redis.exceptions import TimeoutError as RedisTimeoutError + +from litellm.caching.dual_cache import DualCache +from litellm.models.managed_files import LiteLLM_ManagedFileTable +from litellm.proxy.utils import InternalUsageCache, PrismaClient +from litellm.types.llms.openai import OpenAIFileObject +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from tests._support.fake_redis import fake_redis_cache +from tests.unit.enterprise.proxy.hooks.test_managed_files import ( + _InMemoryManagedFilesPrismaClient, # pyright: ignore[reportPrivateUsage] # in-memory doubles shared with the hook suite + _InMemoryManagedFileTable, # pyright: ignore[reportPrivateUsage] # in-memory doubles shared with the hook suite +) + + +async def _stored(raw: "Redis[bytes]", key: str) -> bytes | None: + return await raw.get(key) + + +def _row(file_id: str = "unified-é") -> LiteLLM_ManagedFileTable: + return LiteLLM_ManagedFileTable( + unified_file_id=file_id, + file_object=OpenAIFileObject( + id=file_id, object="file", bytes=123, created_at=456, filename="文.jsonl", purpose="batch" + ), + model_mappings={"model": "provider-file"}, + flat_model_file_ids=["provider-file"], + created_by="owner", + updated_by="owner", + ) + + +def _hook(cache: DualCache, table: _InMemoryManagedFileTable) -> PROXY_LiteLLMManagedFiles: + return PROXY_LiteLLMManagedFiles( + InternalUsageCache(cache), cast(PrismaClient, _InMemoryManagedFilesPrismaClient(table)) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("file_id", ["unified-é", "tenant:unified-é"]) +@pytest.mark.parametrize("ttl", [1, 17.9, None, 0.5, 0, -1]) +async def test_file_metadata_matches_legacy_wire_and_ttl( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, file_id: str, ttl: float | None +) -> None: + cache: Final = fake_redis_cache(server, namespace="tenant") + dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=ttl, default_redis_ttl=999) + dual.update_cache_ttl(ttl, None) + hook: Final = _hook(dual, _InMemoryManagedFileTable()) + row: Final = _row(file_id) + key: Final = file_id if file_id.startswith("tenant:") else f"tenant:{file_id}" + with patch("time.time", return_value=time.time()): + await hook.store_unified_file_id( + file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner") + ) + encoded: Final = await _stored(raw, key) + expiry: Final = await raw.pttl(key) + await cache.async_set_cache("legacy", row.model_dump(), **({"ttl": ttl} if ttl is not None else {})) + assert encoded == await raw.get("tenant:legacy") + assert expiry == await raw.pttl("tenant:legacy") + assert encoded == (None if ttl is not None and ttl < 1 else json.dumps(row.model_dump()).encode()) + assert await cache.async_get_cache(file_id) == (None if ttl is not None and ttl < 1 else row.model_dump()) + assert await raw.get(f"tenant:{key}") is None + + +@pytest.mark.asyncio +async def test_old_writer_new_reader_backfills_only_redis_hits( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + cache: Final = fake_redis_cache(server, namespace="tenant") + dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=1) + row: Final = _row() + table: Final = _InMemoryManagedFileTable(row) + hook: Final = _hook(dual, table) + await cache.async_set_cache(row.unified_file_id, row.model_dump(), ttl=60) + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert table.find_first_calls == [] + server.connected = False + assert await hook.get_unified_file_id(row.unified_file_id) == row + server.connected = True + dual.in_memory_cache.flush_cache() + await raw.delete(f"tenant:{row.unified_file_id}") + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert await raw.get(f"tenant:{row.unified_file_id}") is None + assert len(table.find_first_calls) == 1 + + +@pytest.mark.asyncio +async def test_file_expiry_and_legacy_literal_record( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + row: Final = _row() + with patch("time.time", return_value=1_800_000_000): + cache: Final = fake_redis_cache(server) + table: Final = _InMemoryManagedFileTable(row) + hook: Final = _hook(DualCache(redis_cache=cache, default_in_memory_ttl=1), table) + await raw.set(row.unified_file_id, repr(row.model_dump()), ex=1) + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert table.find_first_calls == [] + with patch("time.time", return_value=1_800_000_002): + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert len(table.find_first_calls) == 1 + assert await raw.get(row.unified_file_id) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("encoded", [b"null", b"{}", b"broken", b"\xff"]) +async def test_file_misses_and_malformed_bytes_fall_back_to_database( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, encoded: bytes +) -> None: + row: Final = _row() + table: Final = _InMemoryManagedFileTable(row) + hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server)), table) + await raw.set(row.unified_file_id, encoded) + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert len(table.find_first_calls) == 1 + assert await raw.get(row.unified_file_id) == encoded + + +@pytest.mark.asyncio +async def test_recursive_file_metadata_retains_legacy_database_fallback( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + row: Final = _row() + table: Final = _InMemoryManagedFileTable(row) + cache: Final = fake_redis_cache(server) + hook: Final = _hook(DualCache(redis_cache=cache), table) + await raw.set(row.unified_file_id, b"[" * 10000 + b"0" + b"]" * 10000) + limit: Final = sys.getrecursionlimit() + try: + sys.setrecursionlimit(1000) + assert await cache.async_get_cache(row.unified_file_id) is None + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert len(table.find_first_calls) == 1 + finally: + sys.setrecursionlimit(limit) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("encoded", [b'{"invalid": true}', b"(1, 2)", b"{'item'}", b"{1: 'value'}"]) +async def test_truthy_invalid_file_record_still_raises_validation_error( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, encoded: bytes +) -> None: + table: Final = _InMemoryManagedFileTable(_row()) + hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server)), table) + await raw.set("unified-é", encoded) + with pytest.raises(ValidationError): + await hook.get_unified_file_id("unified-é") + assert table.find_first_calls == [] + + +class _DeleteObservingTable(_InMemoryManagedFileTable): + def __init__(self, raw: fakeredis.FakeAsyncRedis, row: LiteLLM_ManagedFileTable) -> None: + super().__init__(row) + self.raw = raw + + async def delete(self, where: Mapping[str, object]) -> LiteLLM_ManagedFileTable | None: + assert await self.raw.get(f"tenant:{where['unified_file_id']}") == b"null" + assert await self.raw.pttl(f"tenant:{where['unified_file_id']}") == 1000 + return await super().delete(where) + + +@pytest.mark.asyncio +async def test_file_delete_writes_null_before_database_delete( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + row: Final = _row() + table: Final = _DeleteObservingTable(raw, row) + hook: Final = _hook( + DualCache(redis_cache=fake_redis_cache(server, namespace="tenant"), default_in_memory_ttl=1), table + ) + with patch("time.time", return_value=time.time()): + assert await hook.delete_unified_file_id(row.unified_file_id) == row.file_object + assert table.rows == {} + assert await hook.get_unified_file_id(row.unified_file_id) is None + assert len(table.find_first_calls) == 2 + + +@pytest.mark.asyncio +async def test_file_outage_keeps_local_write_and_database_policy(server: fakeredis.FakeServer) -> None: + row: Final = _row() + cache: Final = fake_redis_cache(server) + table: Final = _InMemoryManagedFileTable() + dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=1) + hook: Final = _hook(dual, table) + server.connected = False + await hook.store_unified_file_id( + row.unified_file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner") + ) + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert table.rows[row.unified_file_id] == row + dual.in_memory_cache.flush_cache() + for _ in range(8): + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert cache._circuit_breaker.state == "open" # pyright: ignore[reportPrivateUsage] # asserts the breaker opened after the outage + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ttl", [float("nan"), float("inf"), float("-inf")]) +async def test_invalid_file_cache_ttl_does_not_block_database_write( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, ttl: float +) -> None: + row: Final = _row() + table: Final = _InMemoryManagedFileTable() + dual: Final = DualCache(redis_cache=fake_redis_cache(server), default_in_memory_ttl=ttl) + hook: Final = _hook(dual, table) + with patch("time.time", return_value=1_800_000_000): + await dual.async_set_cache("legacy", row.model_dump()) + await hook.store_unified_file_id( + row.unified_file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner") + ) + assert table.rows[row.unified_file_id] == row + assert await raw.get(row.unified_file_id) == await raw.get("legacy") + assert await raw.pttl(row.unified_file_id) == await raw.pttl("legacy") + + +@pytest.mark.asyncio +async def test_unknown_applied_file_write_is_not_rolled_back( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + cache: Final = fake_redis_cache(server, namespace="tenant") + row: Final = _row() + table: Final = _InMemoryManagedFileTable() + hook: Final = _hook(DualCache(redis_cache=cache, default_in_memory_ttl=1), table) + + async def applied_then_timeout(name: str, value: str, *, ex: timedelta | None, nx: bool) -> bool: + await raw.set(name, value, ex=ex, nx=nx) + raise RedisTimeoutError("reply lost after apply") + + with patch.object(cache.init_async_client(), "set", side_effect=applied_then_timeout): + await hook.store_unified_file_id( + row.unified_file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner") + ) + assert await raw.get(f"tenant:{row.unified_file_id}") == json.dumps(row.model_dump()).encode() + assert table.rows[row.unified_file_id] == row + + +@pytest.mark.asyncio +async def test_file_refresh_and_lost_compare_and_swap_preserve_cache_order( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + row: Final = _row() + table: Final = _InMemoryManagedFileTable(row) + hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server), default_in_memory_ttl=1), table) + assert row.file_object is not None + updated: Final = row.file_object.model_copy(update={"bytes": 987}) + await hook._save_refreshed_file_object(row, updated) # pyright: ignore[reportPrivateUsage] # exercises the refresh write path + assert await raw.get(row.unified_file_id) == json.dumps(table.rows[row.unified_file_id].model_dump()).encode() + assert await hook._save_xai_refreshed_file_object(row, updated) is None # pyright: ignore[reportPrivateUsage] # exercises the refresh write path + assert await raw.get(row.unified_file_id) == b"null" + assert table.rows[row.unified_file_id].file_object == updated + assert await hook._save_xai_refreshed_file_object(table.rows[row.unified_file_id], updated) == updated # pyright: ignore[reportPrivateUsage] # exercises the refresh write path + assert await raw.get(row.unified_file_id) == json.dumps(table.rows[row.unified_file_id].model_dump()).encode()