refactor(redis): isolate managed file metadata store

This commit is contained in:
Devin AI 2026-10-30 17:50:19 +00:00
parent 8ae309af4e
commit da53bfb788
5 changed files with 341 additions and 38 deletions

View file

@ -36,6 +36,7 @@ from litellm import Router, verbose_logger
from litellm._internal_context import with_service_target
from litellm._uuid import uuid
from litellm.caching.caching import DualCache
from litellm.caching.redis_batch import active_request_redis_batch
from litellm.constants import (
BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS,
BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS,
@ -93,6 +94,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
request_tags_from_metadata,
)
from litellm.redis.stores.managed_files import ManagedFileStore, TransportManagedFileStore, file_key
from litellm.redis.transport.contract import RedisFailure
from litellm.repositories.managed_file_repository import ManagedFileRepository
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
@ -578,6 +581,12 @@ def _keyed_get(source: object, key: str, default: object = None) -> object:
return require_attributes(source, _SupportsKeyedGet, "get").get(key, default)
class _LocalMetadataCache(Protocol):
async def async_get_cache(self, key: str, *, local_only: bool) -> object: ...
async def async_set_cache(self, key: str, value: object, *, local_only: bool) -> None: ...
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# Class variables or attributes
def __init__(
@ -593,6 +602,46 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
self._sleep = sleep
self._clock = clock
def _metadata_cache(self) -> DualCache:
if isinstance(self.internal_usage_cache, DualCache):
return self.internal_usage_cache
return self.internal_usage_cache.dual_cache
async def _cache_file(self, file_id: str, record: Mapping[str, object] | None) -> None:
cache: Final = self._metadata_cache()
local_cache: Final = require_attributes(cache, _LocalMetadataCache, "async_get_cache", "async_set_cache")
await local_cache.async_set_cache(file_key(file_id), record, local_only=True)
if cache.redis_cache is None:
return
store: Final[ManagedFileStore] = TransportManagedFileStore(cache.redis_cache.transport())
seconds: Final = (
int(cache.default_in_memory_ttl)
if cache.default_in_memory_ttl is not None
else cache.redis_cache.default_ttl
)
ttl: Final = timedelta(seconds=seconds)
if record is None:
await store.invalidate_file(file_id, ttl=ttl)
else:
await store.write_file(file_id, record, ttl=ttl)
async def _cached_file(self, file_id: str) -> object:
cache: Final = self._metadata_cache()
local_cache: Final = require_attributes(cache, _LocalMetadataCache, "async_get_cache", "async_set_cache")
local: Final = await local_cache.async_get_cache(file_key(file_id), local_only=True)
if local is not None or cache.redis_cache is None:
return local
request_batch: Final = active_request_redis_batch(cache.redis_cache)
if request_batch is not None and request_batch.read_as_missing(file_key(file_id)):
return None
store: Final[ManagedFileStore] = TransportManagedFileStore(cache.redis_cache.transport())
result: Final = await store.read_file(file_id)
if isinstance(result, RedisFailure):
return None
if result is not None:
await local_cache.async_set_cache(file_key(file_id), result, local_only=True)
return result
@staticmethod
def _get_prometheus_logger():
"""Find PrometheusLogger from litellm.callbacks, if registered."""
@ -623,11 +672,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
storage_backend=storage_metadata.get("storage_backend"),
storage_url=storage_metadata.get("storage_url"),
)
await self.internal_usage_cache.async_set_cache(
key=file_id,
value=litellm_managed_file_object.model_dump(),
litellm_parent_otel_span=litellm_parent_otel_span,
)
await self._cache_file(file_id, litellm_managed_file_object.model_dump())
## STORE MODEL MAPPINGS IN DB
@ -820,13 +865,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
self, file_id: str, litellm_parent_otel_span: Optional[Span] = None
) -> Optional[LiteLLM_ManagedFileTable]:
## CHECK CACHE
result = cast(
Optional[dict],
await self.internal_usage_cache.async_get_cache(
key=file_id,
litellm_parent_otel_span=litellm_parent_otel_span,
),
)
result: Final = await self._cached_file(file_id)
if result:
return LiteLLM_ManagedFileTable.model_validate(result)
@ -847,11 +886,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if initial_value is None:
raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
## delete old value
await self.internal_usage_cache.async_set_cache(
key=file_id,
value=None,
litellm_parent_otel_span=litellm_parent_otel_span,
)
await self._cache_file(file_id, None)
await _managed_file_table(self.prisma_client).delete(where={"unified_file_id": file_id})
return initial_value.file_object
@ -1098,11 +1133,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if not await ManagedFileRepository(self.prisma_client).update_file_object(stored.unified_file_id, file_object):
return
refreshed_row: Final = stored.model_copy(update={"file_object": file_object})
await self.internal_usage_cache.async_set_cache(
key=stored.unified_file_id,
value=refreshed_row.model_dump(),
litellm_parent_otel_span=None,
)
await self._cache_file(stored.unified_file_id, refreshed_row.model_dump())
async def _save_xai_refreshed_file_object(
self,
@ -1117,14 +1148,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
sized_file_object,
if_file_object=_stored_json_text(stored.file_object) if stored.file_object is not None else None,
):
await self.internal_usage_cache.async_set_cache(
key=stored.unified_file_id, value=None, litellm_parent_otel_span=None
)
await self._cache_file(stored.unified_file_id, None)
return None
await self.internal_usage_cache.async_set_cache(
key=stored.unified_file_id,
value=stored.model_copy(update={"file_object": sized_file_object}).model_dump(),
litellm_parent_otel_span=None,
await self._cache_file(
stored.unified_file_id, stored.model_copy(update={"file_object": sized_file_object}).model_dump()
)
return sized_file_object

View file

@ -0,0 +1,61 @@
import ast
import json
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import timedelta
from typing import Final, Protocol
from pydantic import JsonValue, TypeAdapter
from litellm.redis.transport.contract import RedisFailure, RedisTransport
_RECORD: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
def file_key(unified_file_id: str) -> str:
return unified_file_id
class ManagedFileStore(Protocol):
async def read_file(self, unified_file_id: str) -> JsonValue | RedisFailure: ...
async def write_file(
self, unified_file_id: str, record: Mapping[str, object], *, ttl: timedelta | None
) -> None | RedisFailure: ...
async def invalidate_file(self, unified_file_id: str, *, ttl: timedelta | None) -> None | RedisFailure: ...
@dataclass(frozen=True, slots=True)
class TransportManagedFileStore:
transport: RedisTransport
async def read_file(self, unified_file_id: str) -> JsonValue | RedisFailure:
raw: Final = await self.transport.get(file_key(unified_file_id))
if raw is None or isinstance(raw, RedisFailure):
return raw
try:
return _RECORD.validate_python(json.loads(raw.decode("utf-8")))
except (ValueError, UnicodeError):
try:
return _RECORD.validate_python(ast.literal_eval(raw.decode("utf-8")))
except (ValueError, SyntaxError, UnicodeError) as exc:
return RedisFailure(kind="decode", message=str(exc))
async def write_file(
self, unified_file_id: str, record: Mapping[str, object], *, ttl: timedelta | None
) -> None | RedisFailure:
try:
encoded: Final = json.dumps(record)
except (TypeError, ValueError) as exc:
return RedisFailure(kind="decode", message=str(exc))
return await self._write(unified_file_id, encoded, ttl)
async def invalidate_file(self, unified_file_id: str, *, ttl: timedelta | None) -> None | RedisFailure:
return await self._write(unified_file_id, "null", ttl)
async def _write(self, unified_file_id: str, encoded: str, ttl: timedelta | None) -> None | RedisFailure:
if ttl is not None and ttl <= timedelta(0):
return RedisFailure(kind="server", message="invalid expire time in set")
result: Final = await self.transport.set(file_key(unified_file_id), encoded, ttl=ttl)
return result if isinstance(result, RedisFailure) else None

View file

@ -1783,8 +1783,7 @@ async def test_store_unified_file_id_updates_file_metadata_on_existing_row():
prisma_client.db.litellm_managedfiletable.upsert = AsyncMock(
return_value=MagicMock()
)
internal_usage_cache = MagicMock()
internal_usage_cache.async_set_cache = AsyncMock()
internal_usage_cache = DualCache()
proxy_managed_files = PROXY_LiteLLMManagedFiles(
internal_usage_cache=internal_usage_cache,
@ -3404,9 +3403,10 @@ async def test_afile_delete_returns_provider_response_when_stored_file_object_no
)
prisma_client.db.litellm_managedfiletable.delete = AsyncMock()
internal_usage_cache = MagicMock()
internal_usage_cache.async_get_cache = AsyncMock(
return_value={
internal_usage_cache = DualCache()
await internal_usage_cache.async_set_cache(
key=unified_file_id,
value={
"unified_file_id": unified_file_id,
"model_mappings": {"model-123": "file-provider-xyz"},
"flat_model_file_ids": ["file-provider-xyz"],
@ -3415,7 +3415,6 @@ async def test_afile_delete_returns_provider_response_when_stored_file_object_no
"updated_by": "test-user",
}
)
internal_usage_cache.async_set_cache = AsyncMock()
proxy_managed_files = PROXY_LiteLLMManagedFiles(
internal_usage_cache=internal_usage_cache,

View file

@ -1402,9 +1402,9 @@ def _make_real_managed_files_instance():
PROXY_LiteLLMManagedFiles,
)
mock_cache = MagicMock()
mock_cache.async_get_cache = AsyncMock(return_value=None)
mock_cache.async_set_cache = AsyncMock()
from litellm.caching.dual_cache import DualCache
mock_cache = DualCache()
mock_prisma = MagicMock()
mock_prisma.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None)

View file

@ -0,0 +1,216 @@
import json
import time
from collections.abc import Mapping
from typing import Final, cast
from unittest.mock import patch
import fakeredis
import pytest
from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles
from pydantic import ValidationError
from redis.exceptions import TimeoutError as RedisTimeoutError
from litellm.caching.dual_cache import DualCache
from litellm.models.managed_files import LiteLLM_ManagedFileTable
from litellm.proxy.utils import InternalUsageCache, PrismaClient
from litellm.types.llms.openai import OpenAIFileObject
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from tests._support.fake_redis import fake_redis_cache
from tests.unit.enterprise.proxy.hooks.test_managed_files import (
_InMemoryManagedFilesPrismaClient,
_InMemoryManagedFileTable,
)
def _row(file_id: str = "unified-é") -> LiteLLM_ManagedFileTable:
return LiteLLM_ManagedFileTable(
unified_file_id=file_id,
file_object=OpenAIFileObject(
id=file_id, object="file", bytes=123, created_at=456, filename="文.jsonl", purpose="batch"
),
model_mappings={"model": "provider-file"},
flat_model_file_ids=["provider-file"],
created_by="owner",
updated_by="owner",
)
def _hook(cache: DualCache, table: _InMemoryManagedFileTable) -> PROXY_LiteLLMManagedFiles:
return PROXY_LiteLLMManagedFiles(
InternalUsageCache(cache), cast(PrismaClient, _InMemoryManagedFilesPrismaClient(table))
)
@pytest.mark.asyncio
@pytest.mark.parametrize("file_id", ["unified-é", "tenant:unified-é"])
@pytest.mark.parametrize("ttl", [1, 17.9, None, 0.5])
async def test_file_metadata_matches_legacy_wire_and_ttl(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, file_id: str, ttl: float | None
) -> None:
cache: Final = fake_redis_cache(server, namespace="tenant")
dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=ttl, default_redis_ttl=999)
hook: Final = _hook(dual, _InMemoryManagedFileTable())
row: Final = _row(file_id)
key: Final = file_id if file_id.startswith("tenant:") else f"tenant:{file_id}"
with patch("time.time", return_value=time.time()):
await hook.store_unified_file_id(
file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner")
)
encoded: Final = await raw.get(key)
expiry: Final = await raw.pttl(key)
await cache.async_set_cache("legacy", row.model_dump(), **({"ttl": ttl} if ttl is not None else {}))
assert encoded == await raw.get("tenant:legacy")
assert expiry == await raw.pttl("tenant:legacy")
assert encoded == (None if ttl == 0.5 else json.dumps(row.model_dump()).encode())
assert await cache.async_get_cache(file_id) == (None if ttl == 0.5 else row.model_dump())
assert await raw.get(f"tenant:{key}") is None
@pytest.mark.asyncio
async def test_old_writer_new_reader_backfills_only_redis_hits(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
cache: Final = fake_redis_cache(server, namespace="tenant")
dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=1)
row: Final = _row()
table: Final = _InMemoryManagedFileTable(row)
hook: Final = _hook(dual, table)
await cache.async_set_cache(row.unified_file_id, row.model_dump(), ttl=60)
assert await hook.get_unified_file_id(row.unified_file_id) == row
assert table.find_first_calls == []
server.connected = False
assert await hook.get_unified_file_id(row.unified_file_id) == row
server.connected = True
dual.in_memory_cache.flush_cache()
await raw.delete(f"tenant:{row.unified_file_id}")
assert await hook.get_unified_file_id(row.unified_file_id) == row
assert await raw.get(f"tenant:{row.unified_file_id}") is None
assert len(table.find_first_calls) == 1
@pytest.mark.asyncio
async def test_file_expiry_and_legacy_literal_record(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
row: Final = _row()
with patch("time.time", return_value=1_800_000_000):
cache: Final = fake_redis_cache(server)
table: Final = _InMemoryManagedFileTable(row)
hook: Final = _hook(DualCache(redis_cache=cache, default_in_memory_ttl=1), table)
await raw.set(row.unified_file_id, repr(row.model_dump()), ex=1)
assert await hook.get_unified_file_id(row.unified_file_id) == row
assert table.find_first_calls == []
with patch("time.time", return_value=1_800_000_002):
assert await hook.get_unified_file_id(row.unified_file_id) == row
assert len(table.find_first_calls) == 1
assert await raw.get(row.unified_file_id) is None
@pytest.mark.asyncio
@pytest.mark.parametrize("encoded", [b"null", b"{}", b"broken", b"\xff"])
async def test_file_misses_and_malformed_bytes_fall_back_to_database(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, encoded: bytes
) -> None:
row: Final = _row()
table: Final = _InMemoryManagedFileTable(row)
hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server)), table)
await raw.set(row.unified_file_id, encoded)
assert await hook.get_unified_file_id(row.unified_file_id) == row
assert len(table.find_first_calls) == 1
assert await raw.get(row.unified_file_id) == encoded
@pytest.mark.asyncio
async def test_truthy_invalid_file_record_still_raises_validation_error(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
table: Final = _InMemoryManagedFileTable(_row())
hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server)), table)
await raw.set("unified-é", b'{"invalid": true}')
with pytest.raises(ValidationError):
await hook.get_unified_file_id("unified-é")
assert table.find_first_calls == []
class _DeleteObservingTable(_InMemoryManagedFileTable):
def __init__(self, raw: fakeredis.FakeAsyncRedis, row: LiteLLM_ManagedFileTable) -> None:
super().__init__(row)
self.raw = raw
async def delete(self, where: Mapping[str, object]) -> LiteLLM_ManagedFileTable | None:
assert await self.raw.get(f"tenant:{where['unified_file_id']}") == b"null"
assert await self.raw.pttl(f"tenant:{where['unified_file_id']}") == 1000
return await super().delete(where)
@pytest.mark.asyncio
async def test_file_delete_writes_null_before_database_delete(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
row: Final = _row()
table: Final = _DeleteObservingTable(raw, row)
hook: Final = _hook(
DualCache(redis_cache=fake_redis_cache(server, namespace="tenant"), default_in_memory_ttl=1), table
)
with patch("time.time", return_value=time.time()):
assert await hook.delete_unified_file_id(row.unified_file_id) == row.file_object
assert table.rows == {}
assert await hook.get_unified_file_id(row.unified_file_id) is None
assert len(table.find_first_calls) == 2
@pytest.mark.asyncio
async def test_file_outage_keeps_local_write_and_database_policy(server: fakeredis.FakeServer) -> None:
row: Final = _row()
cache: Final = fake_redis_cache(server)
table: Final = _InMemoryManagedFileTable()
dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=1)
hook: Final = _hook(dual, table)
server.connected = False
await hook.store_unified_file_id(
row.unified_file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner")
)
assert await hook.get_unified_file_id(row.unified_file_id) == row
assert table.rows[row.unified_file_id] == row
dual.in_memory_cache.flush_cache()
for _ in range(8):
assert await hook.get_unified_file_id(row.unified_file_id) == row
assert cache._circuit_breaker.state == "open"
@pytest.mark.asyncio
async def test_unknown_applied_file_write_is_not_rolled_back(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
cache: Final = fake_redis_cache(server, namespace="tenant")
row: Final = _row()
table: Final = _InMemoryManagedFileTable()
hook: Final = _hook(DualCache(redis_cache=cache, default_in_memory_ttl=1), table)
async def applied_then_timeout(name: str, value: str, *, px: int | None, nx: bool) -> bool:
await raw.set(name, value, px=px, nx=nx)
raise RedisTimeoutError("reply lost after apply")
with patch.object(cache.init_async_client(), "set", side_effect=applied_then_timeout):
await hook.store_unified_file_id(
row.unified_file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner")
)
assert await raw.get(f"tenant:{row.unified_file_id}") == json.dumps(row.model_dump()).encode()
assert table.rows[row.unified_file_id] == row
@pytest.mark.asyncio
async def test_file_refresh_and_lost_compare_and_swap_preserve_cache_order(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
row: Final = _row()
table: Final = _InMemoryManagedFileTable(row)
hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server), default_in_memory_ttl=1), table)
updated: Final = row.file_object.model_copy(update={"bytes": 987})
await hook._save_refreshed_file_object(row, updated)
assert await raw.get(row.unified_file_id) == json.dumps(table.rows[row.unified_file_id].model_dump()).encode()
assert await hook._save_xai_refreshed_file_object(row, updated) is None
assert await raw.get(row.unified_file_id) == b"null"
assert table.rows[row.unified_file_id].file_object == updated
assert await hook._save_xai_refreshed_file_object(table.rows[row.unified_file_id], updated) == updated
assert await raw.get(row.unified_file_id) == json.dumps(table.rows[row.unified_file_id].model_dump()).encode()