From da53bfb7880756358c4a60687dc5f987fb474abe Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:50:19 +0000 Subject: [PATCH] refactor(redis): isolate managed file metadata store --- .../proxy/hooks/managed_files.py | 85 ++++--- litellm/redis/stores/managed_files.py | 61 +++++ .../proxy/hooks/test_managed_files.py | 11 +- .../proxy/test_managed_files_hook.py | 6 +- tests/unit/redis/stores/test_managed_files.py | 216 ++++++++++++++++++ 5 files changed, 341 insertions(+), 38 deletions(-) create mode 100644 litellm/redis/stores/managed_files.py create mode 100644 tests/unit/redis/stores/test_managed_files.py diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 5b54e7d72a0..2266478d7ea 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -36,6 +36,7 @@ from litellm import Router, verbose_logger from litellm._internal_context import with_service_target from litellm._uuid import uuid from litellm.caching.caching import DualCache +from litellm.caching.redis_batch import active_request_redis_batch from litellm.constants import ( BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS, BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS, @@ -93,6 +94,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( request_tags_from_metadata, ) +from litellm.redis.stores.managed_files import ManagedFileStore, TransportManagedFileStore, file_key +from litellm.redis.transport.contract import RedisFailure from litellm.repositories.managed_file_repository import ManagedFileRepository from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue] @@ -578,6 +581,12 @@ def _keyed_get(source: object, key: str, default: object = None) -> object: return require_attributes(source, _SupportsKeyedGet, "get").get(key, default) +class _LocalMetadataCache(Protocol): + async def async_get_cache(self, key: str, *, local_only: bool) -> object: ... + + async def async_set_cache(self, key: str, value: object, *, local_only: bool) -> None: ... + + class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): # Class variables or attributes def __init__( @@ -593,6 +602,46 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self._sleep = sleep self._clock = clock + def _metadata_cache(self) -> DualCache: + if isinstance(self.internal_usage_cache, DualCache): + return self.internal_usage_cache + return self.internal_usage_cache.dual_cache + + async def _cache_file(self, file_id: str, record: Mapping[str, object] | None) -> None: + cache: Final = self._metadata_cache() + local_cache: Final = require_attributes(cache, _LocalMetadataCache, "async_get_cache", "async_set_cache") + await local_cache.async_set_cache(file_key(file_id), record, local_only=True) + if cache.redis_cache is None: + return + store: Final[ManagedFileStore] = TransportManagedFileStore(cache.redis_cache.transport()) + seconds: Final = ( + int(cache.default_in_memory_ttl) + if cache.default_in_memory_ttl is not None + else cache.redis_cache.default_ttl + ) + ttl: Final = timedelta(seconds=seconds) + if record is None: + await store.invalidate_file(file_id, ttl=ttl) + else: + await store.write_file(file_id, record, ttl=ttl) + + async def _cached_file(self, file_id: str) -> object: + cache: Final = self._metadata_cache() + local_cache: Final = require_attributes(cache, _LocalMetadataCache, "async_get_cache", "async_set_cache") + local: Final = await local_cache.async_get_cache(file_key(file_id), local_only=True) + if local is not None or cache.redis_cache is None: + return local + request_batch: Final = active_request_redis_batch(cache.redis_cache) + if request_batch is not None and request_batch.read_as_missing(file_key(file_id)): + return None + store: Final[ManagedFileStore] = TransportManagedFileStore(cache.redis_cache.transport()) + result: Final = await store.read_file(file_id) + if isinstance(result, RedisFailure): + return None + if result is not None: + await local_cache.async_set_cache(file_key(file_id), result, local_only=True) + return result + @staticmethod def _get_prometheus_logger(): """Find PrometheusLogger from litellm.callbacks, if registered.""" @@ -623,11 +672,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): storage_backend=storage_metadata.get("storage_backend"), storage_url=storage_metadata.get("storage_url"), ) - await self.internal_usage_cache.async_set_cache( - key=file_id, - value=litellm_managed_file_object.model_dump(), - litellm_parent_otel_span=litellm_parent_otel_span, - ) + await self._cache_file(file_id, litellm_managed_file_object.model_dump()) ## STORE MODEL MAPPINGS IN DB @@ -820,13 +865,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self, file_id: str, litellm_parent_otel_span: Optional[Span] = None ) -> Optional[LiteLLM_ManagedFileTable]: ## CHECK CACHE - result = cast( - Optional[dict], - await self.internal_usage_cache.async_get_cache( - key=file_id, - litellm_parent_otel_span=litellm_parent_otel_span, - ), - ) + result: Final = await self._cached_file(file_id) if result: return LiteLLM_ManagedFileTable.model_validate(result) @@ -847,11 +886,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if initial_value is None: raise Exception(f"LiteLLM Managed File object with id={file_id} not found") ## delete old value - await self.internal_usage_cache.async_set_cache( - key=file_id, - value=None, - litellm_parent_otel_span=litellm_parent_otel_span, - ) + await self._cache_file(file_id, None) await _managed_file_table(self.prisma_client).delete(where={"unified_file_id": file_id}) return initial_value.file_object @@ -1098,11 +1133,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if not await ManagedFileRepository(self.prisma_client).update_file_object(stored.unified_file_id, file_object): return refreshed_row: Final = stored.model_copy(update={"file_object": file_object}) - await self.internal_usage_cache.async_set_cache( - key=stored.unified_file_id, - value=refreshed_row.model_dump(), - litellm_parent_otel_span=None, - ) + await self._cache_file(stored.unified_file_id, refreshed_row.model_dump()) async def _save_xai_refreshed_file_object( self, @@ -1117,14 +1148,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): sized_file_object, if_file_object=_stored_json_text(stored.file_object) if stored.file_object is not None else None, ): - await self.internal_usage_cache.async_set_cache( - key=stored.unified_file_id, value=None, litellm_parent_otel_span=None - ) + await self._cache_file(stored.unified_file_id, None) return None - await self.internal_usage_cache.async_set_cache( - key=stored.unified_file_id, - value=stored.model_copy(update={"file_object": sized_file_object}).model_dump(), - litellm_parent_otel_span=None, + await self._cache_file( + stored.unified_file_id, stored.model_copy(update={"file_object": sized_file_object}).model_dump() ) return sized_file_object diff --git a/litellm/redis/stores/managed_files.py b/litellm/redis/stores/managed_files.py new file mode 100644 index 00000000000..594cb9723c8 --- /dev/null +++ b/litellm/redis/stores/managed_files.py @@ -0,0 +1,61 @@ +import ast +import json +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import timedelta +from typing import Final, Protocol + +from pydantic import JsonValue, TypeAdapter + +from litellm.redis.transport.contract import RedisFailure, RedisTransport + +_RECORD: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def file_key(unified_file_id: str) -> str: + return unified_file_id + + +class ManagedFileStore(Protocol): + async def read_file(self, unified_file_id: str) -> JsonValue | RedisFailure: ... + + async def write_file( + self, unified_file_id: str, record: Mapping[str, object], *, ttl: timedelta | None + ) -> None | RedisFailure: ... + + async def invalidate_file(self, unified_file_id: str, *, ttl: timedelta | None) -> None | RedisFailure: ... + + +@dataclass(frozen=True, slots=True) +class TransportManagedFileStore: + transport: RedisTransport + + async def read_file(self, unified_file_id: str) -> JsonValue | RedisFailure: + raw: Final = await self.transport.get(file_key(unified_file_id)) + if raw is None or isinstance(raw, RedisFailure): + return raw + try: + return _RECORD.validate_python(json.loads(raw.decode("utf-8"))) + except (ValueError, UnicodeError): + try: + return _RECORD.validate_python(ast.literal_eval(raw.decode("utf-8"))) + except (ValueError, SyntaxError, UnicodeError) as exc: + return RedisFailure(kind="decode", message=str(exc)) + + async def write_file( + self, unified_file_id: str, record: Mapping[str, object], *, ttl: timedelta | None + ) -> None | RedisFailure: + try: + encoded: Final = json.dumps(record) + except (TypeError, ValueError) as exc: + return RedisFailure(kind="decode", message=str(exc)) + return await self._write(unified_file_id, encoded, ttl) + + async def invalidate_file(self, unified_file_id: str, *, ttl: timedelta | None) -> None | RedisFailure: + return await self._write(unified_file_id, "null", ttl) + + async def _write(self, unified_file_id: str, encoded: str, ttl: timedelta | None) -> None | RedisFailure: + if ttl is not None and ttl <= timedelta(0): + return RedisFailure(kind="server", message="invalid expire time in set") + result: Final = await self.transport.set(file_key(unified_file_id), encoded, ttl=ttl) + return result if isinstance(result, RedisFailure) else None diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index 1755ab97689..81302a5f4f4 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -1783,8 +1783,7 @@ async def test_store_unified_file_id_updates_file_metadata_on_existing_row(): prisma_client.db.litellm_managedfiletable.upsert = AsyncMock( return_value=MagicMock() ) - internal_usage_cache = MagicMock() - internal_usage_cache.async_set_cache = AsyncMock() + internal_usage_cache = DualCache() proxy_managed_files = PROXY_LiteLLMManagedFiles( internal_usage_cache=internal_usage_cache, @@ -3404,9 +3403,10 @@ async def test_afile_delete_returns_provider_response_when_stored_file_object_no ) prisma_client.db.litellm_managedfiletable.delete = AsyncMock() - internal_usage_cache = MagicMock() - internal_usage_cache.async_get_cache = AsyncMock( - return_value={ + internal_usage_cache = DualCache() + await internal_usage_cache.async_set_cache( + key=unified_file_id, + value={ "unified_file_id": unified_file_id, "model_mappings": {"model-123": "file-provider-xyz"}, "flat_model_file_ids": ["file-provider-xyz"], @@ -3415,7 +3415,6 @@ async def test_afile_delete_returns_provider_response_when_stored_file_object_no "updated_by": "test-user", } ) - internal_usage_cache.async_set_cache = AsyncMock() proxy_managed_files = PROXY_LiteLLMManagedFiles( internal_usage_cache=internal_usage_cache, diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index f2df14cc8a3..88f43bb248a 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -1402,9 +1402,9 @@ def _make_real_managed_files_instance(): PROXY_LiteLLMManagedFiles, ) - mock_cache = MagicMock() - mock_cache.async_get_cache = AsyncMock(return_value=None) - mock_cache.async_set_cache = AsyncMock() + from litellm.caching.dual_cache import DualCache + + mock_cache = DualCache() mock_prisma = MagicMock() mock_prisma.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None) diff --git a/tests/unit/redis/stores/test_managed_files.py b/tests/unit/redis/stores/test_managed_files.py new file mode 100644 index 00000000000..6e20fa4dda2 --- /dev/null +++ b/tests/unit/redis/stores/test_managed_files.py @@ -0,0 +1,216 @@ +import json +import time +from collections.abc import Mapping +from typing import Final, cast +from unittest.mock import patch + +import fakeredis +import pytest +from litellm_enterprise.proxy.hooks.managed_files import PROXY_LiteLLMManagedFiles +from pydantic import ValidationError +from redis.exceptions import TimeoutError as RedisTimeoutError + +from litellm.caching.dual_cache import DualCache +from litellm.models.managed_files import LiteLLM_ManagedFileTable +from litellm.proxy.utils import InternalUsageCache, PrismaClient +from litellm.types.llms.openai import OpenAIFileObject +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from tests._support.fake_redis import fake_redis_cache +from tests.unit.enterprise.proxy.hooks.test_managed_files import ( + _InMemoryManagedFilesPrismaClient, + _InMemoryManagedFileTable, +) + + +def _row(file_id: str = "unified-é") -> LiteLLM_ManagedFileTable: + return LiteLLM_ManagedFileTable( + unified_file_id=file_id, + file_object=OpenAIFileObject( + id=file_id, object="file", bytes=123, created_at=456, filename="文.jsonl", purpose="batch" + ), + model_mappings={"model": "provider-file"}, + flat_model_file_ids=["provider-file"], + created_by="owner", + updated_by="owner", + ) + + +def _hook(cache: DualCache, table: _InMemoryManagedFileTable) -> PROXY_LiteLLMManagedFiles: + return PROXY_LiteLLMManagedFiles( + InternalUsageCache(cache), cast(PrismaClient, _InMemoryManagedFilesPrismaClient(table)) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("file_id", ["unified-é", "tenant:unified-é"]) +@pytest.mark.parametrize("ttl", [1, 17.9, None, 0.5]) +async def test_file_metadata_matches_legacy_wire_and_ttl( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, file_id: str, ttl: float | None +) -> None: + cache: Final = fake_redis_cache(server, namespace="tenant") + dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=ttl, default_redis_ttl=999) + hook: Final = _hook(dual, _InMemoryManagedFileTable()) + row: Final = _row(file_id) + key: Final = file_id if file_id.startswith("tenant:") else f"tenant:{file_id}" + with patch("time.time", return_value=time.time()): + await hook.store_unified_file_id( + file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner") + ) + encoded: Final = await raw.get(key) + expiry: Final = await raw.pttl(key) + await cache.async_set_cache("legacy", row.model_dump(), **({"ttl": ttl} if ttl is not None else {})) + assert encoded == await raw.get("tenant:legacy") + assert expiry == await raw.pttl("tenant:legacy") + assert encoded == (None if ttl == 0.5 else json.dumps(row.model_dump()).encode()) + assert await cache.async_get_cache(file_id) == (None if ttl == 0.5 else row.model_dump()) + assert await raw.get(f"tenant:{key}") is None + + +@pytest.mark.asyncio +async def test_old_writer_new_reader_backfills_only_redis_hits( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + cache: Final = fake_redis_cache(server, namespace="tenant") + dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=1) + row: Final = _row() + table: Final = _InMemoryManagedFileTable(row) + hook: Final = _hook(dual, table) + await cache.async_set_cache(row.unified_file_id, row.model_dump(), ttl=60) + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert table.find_first_calls == [] + server.connected = False + assert await hook.get_unified_file_id(row.unified_file_id) == row + server.connected = True + dual.in_memory_cache.flush_cache() + await raw.delete(f"tenant:{row.unified_file_id}") + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert await raw.get(f"tenant:{row.unified_file_id}") is None + assert len(table.find_first_calls) == 1 + + +@pytest.mark.asyncio +async def test_file_expiry_and_legacy_literal_record( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + row: Final = _row() + with patch("time.time", return_value=1_800_000_000): + cache: Final = fake_redis_cache(server) + table: Final = _InMemoryManagedFileTable(row) + hook: Final = _hook(DualCache(redis_cache=cache, default_in_memory_ttl=1), table) + await raw.set(row.unified_file_id, repr(row.model_dump()), ex=1) + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert table.find_first_calls == [] + with patch("time.time", return_value=1_800_000_002): + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert len(table.find_first_calls) == 1 + assert await raw.get(row.unified_file_id) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("encoded", [b"null", b"{}", b"broken", b"\xff"]) +async def test_file_misses_and_malformed_bytes_fall_back_to_database( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, encoded: bytes +) -> None: + row: Final = _row() + table: Final = _InMemoryManagedFileTable(row) + hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server)), table) + await raw.set(row.unified_file_id, encoded) + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert len(table.find_first_calls) == 1 + assert await raw.get(row.unified_file_id) == encoded + + +@pytest.mark.asyncio +async def test_truthy_invalid_file_record_still_raises_validation_error( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + table: Final = _InMemoryManagedFileTable(_row()) + hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server)), table) + await raw.set("unified-é", b'{"invalid": true}') + with pytest.raises(ValidationError): + await hook.get_unified_file_id("unified-é") + assert table.find_first_calls == [] + + +class _DeleteObservingTable(_InMemoryManagedFileTable): + def __init__(self, raw: fakeredis.FakeAsyncRedis, row: LiteLLM_ManagedFileTable) -> None: + super().__init__(row) + self.raw = raw + + async def delete(self, where: Mapping[str, object]) -> LiteLLM_ManagedFileTable | None: + assert await self.raw.get(f"tenant:{where['unified_file_id']}") == b"null" + assert await self.raw.pttl(f"tenant:{where['unified_file_id']}") == 1000 + return await super().delete(where) + + +@pytest.mark.asyncio +async def test_file_delete_writes_null_before_database_delete( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + row: Final = _row() + table: Final = _DeleteObservingTable(raw, row) + hook: Final = _hook( + DualCache(redis_cache=fake_redis_cache(server, namespace="tenant"), default_in_memory_ttl=1), table + ) + with patch("time.time", return_value=time.time()): + assert await hook.delete_unified_file_id(row.unified_file_id) == row.file_object + assert table.rows == {} + assert await hook.get_unified_file_id(row.unified_file_id) is None + assert len(table.find_first_calls) == 2 + + +@pytest.mark.asyncio +async def test_file_outage_keeps_local_write_and_database_policy(server: fakeredis.FakeServer) -> None: + row: Final = _row() + cache: Final = fake_redis_cache(server) + table: Final = _InMemoryManagedFileTable() + dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=1) + hook: Final = _hook(dual, table) + server.connected = False + await hook.store_unified_file_id( + row.unified_file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner") + ) + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert table.rows[row.unified_file_id] == row + dual.in_memory_cache.flush_cache() + for _ in range(8): + assert await hook.get_unified_file_id(row.unified_file_id) == row + assert cache._circuit_breaker.state == "open" + + +@pytest.mark.asyncio +async def test_unknown_applied_file_write_is_not_rolled_back( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + cache: Final = fake_redis_cache(server, namespace="tenant") + row: Final = _row() + table: Final = _InMemoryManagedFileTable() + hook: Final = _hook(DualCache(redis_cache=cache, default_in_memory_ttl=1), table) + + async def applied_then_timeout(name: str, value: str, *, px: int | None, nx: bool) -> bool: + await raw.set(name, value, px=px, nx=nx) + raise RedisTimeoutError("reply lost after apply") + + with patch.object(cache.init_async_client(), "set", side_effect=applied_then_timeout): + await hook.store_unified_file_id( + row.unified_file_id, row.file_object, None, row.model_mappings, UserAPIKeyAuth(user_id="owner") + ) + assert await raw.get(f"tenant:{row.unified_file_id}") == json.dumps(row.model_dump()).encode() + assert table.rows[row.unified_file_id] == row + + +@pytest.mark.asyncio +async def test_file_refresh_and_lost_compare_and_swap_preserve_cache_order( + server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis +) -> None: + row: Final = _row() + table: Final = _InMemoryManagedFileTable(row) + hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server), default_in_memory_ttl=1), table) + updated: Final = row.file_object.model_copy(update={"bytes": 987}) + await hook._save_refreshed_file_object(row, updated) + assert await raw.get(row.unified_file_id) == json.dumps(table.rows[row.unified_file_id].model_dump()).encode() + assert await hook._save_xai_refreshed_file_object(row, updated) is None + assert await raw.get(row.unified_file_id) == b"null" + assert table.rows[row.unified_file_id].file_object == updated + assert await hook._save_xai_refreshed_file_object(table.rows[row.unified_file_id], updated) == updated + assert await raw.get(row.unified_file_id) == json.dumps(table.rows[row.unified_file_id].model_dump()).encode()