refactor(redis): isolate managed vector resource metadata store

This commit is contained in:
Devin AI 2026-10-30 17:51:42 +00:00
parent da53bfb788
commit dee4ebe96a
4 changed files with 353 additions and 17 deletions

View file

@ -6,16 +6,26 @@ import base64
import json
from abc import ABC, abstractmethod
from collections.abc import Mapping
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Final, Generic, Protocol, TypeVar, cast, runtime_checkable
from litellm import verbose_logger
from litellm._internal_context import with_service_target
from litellm.caching.dual_cache import DualCache
from litellm.caching.redis_batch import active_request_redis_batch
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.llms.base_llm.managed_resources.isolation import (
build_list_page,
build_owner_filter,
can_access_resource,
resolve_resource_owner_id,
)
from litellm.redis.stores.managed_vector_resources import (
ManagedVectorResourceStore,
TransportManagedVectorResourceStore,
vector_resource_key,
)
from litellm.redis.transport.contract import RedisFailure
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.types.utils import SpecialEnums
@ -66,6 +76,12 @@ class _ManagedResourceTable(Protocol[ResourceObjectType]):
async def delete(self, *, where: Mapping[str, object]) -> object: ...
class _LocalVectorResourceCache(Protocol):
async def async_get_cache(self, key: str, *, local_only: bool) -> object: ...
async def async_set_cache(self, key: str, value: object, *, local_only: bool) -> None: ...
class BaseManagedResource(ABC, Generic[ResourceObjectType]):
"""
Base class for managing resources with target_model_names support.
@ -95,6 +111,46 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
def _resource_table(self) -> _ManagedResourceTable[ResourceObjectType]:
return getattr(self.prisma_client.db, self.table_name)
def _metadata_cache(self) -> DualCache:
if isinstance(self.internal_usage_cache, DualCache):
return self.internal_usage_cache
return self.internal_usage_cache.dual_cache
async def _cache_vector_resource(self, unified_resource_id: str, record: Mapping[str, object] | None) -> None:
cache: Final = self._metadata_cache()
local_cache: Final = require_attributes(cache, _LocalVectorResourceCache, "async_get_cache", "async_set_cache")
await local_cache.async_set_cache(vector_resource_key(unified_resource_id), record, local_only=True)
if cache.redis_cache is None:
return
store: Final[ManagedVectorResourceStore] = TransportManagedVectorResourceStore(cache.redis_cache.transport())
seconds: Final = (
int(cache.default_in_memory_ttl)
if cache.default_in_memory_ttl is not None
else cache.redis_cache.default_ttl
)
ttl: Final = timedelta(seconds=seconds)
if record is None:
await store.invalidate_vector_resource(unified_resource_id, ttl=ttl)
else:
await store.write_vector_resource(unified_resource_id, record, ttl=ttl)
async def _cached_vector_resource(self, unified_resource_id: str) -> object:
cache: Final = self._metadata_cache()
local_cache: Final = require_attributes(cache, _LocalVectorResourceCache, "async_get_cache", "async_set_cache")
local: Final = await local_cache.async_get_cache(vector_resource_key(unified_resource_id), local_only=True)
if local is not None or cache.redis_cache is None:
return local
request_batch: Final = active_request_redis_batch(cache.redis_cache)
if request_batch is not None and request_batch.read_as_missing(vector_resource_key(unified_resource_id)):
return None
store: Final[ManagedVectorResourceStore] = TransportManagedVectorResourceStore(cache.redis_cache.transport())
result: Final = await store.read_vector_resource(unified_resource_id)
if isinstance(result, RedisFailure):
return None
if result is not None:
await local_cache.async_set_cache(vector_resource_key(unified_resource_id), result, local_only=True)
return result
# ============================================================================
# ABSTRACT METHODS
# ============================================================================
@ -201,11 +257,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
# Store in cache
if resource_object is not None:
await self.internal_usage_cache.async_set_cache(
key=unified_resource_id,
value=cache_data,
litellm_parent_otel_span=litellm_parent_otel_span,
)
await self._cache_vector_resource(unified_resource_id, cache_data)
# Prepare database data
db_data: Final[dict[str, object]] = {
@ -263,10 +315,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
# Check cache first
result: Final = cast(
dict | None,
await self.internal_usage_cache.async_get_cache(
key=unified_resource_id,
litellm_parent_otel_span=litellm_parent_otel_span,
),
await self._cached_vector_resource(unified_resource_id),
)
if result:
@ -305,11 +354,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
raise Exception(f"LiteLLM Managed {self.resource_type} with id={unified_resource_id} not found")
# Delete from cache
await self.internal_usage_cache.async_set_cache(
key=unified_resource_id,
value=None,
litellm_parent_otel_span=litellm_parent_otel_span,
)
await self._cache_vector_resource(unified_resource_id, None)
# Delete from database
await table.delete(where={"unified_resource_id": unified_resource_id})

View file

@ -0,0 +1,65 @@
import ast
import json
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import timedelta
from typing import Final, Protocol
from pydantic import JsonValue, TypeAdapter
from litellm.redis.transport.contract import RedisFailure, RedisTransport
_RECORD: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
def vector_resource_key(unified_resource_id: str) -> str:
return unified_resource_id
class ManagedVectorResourceStore(Protocol):
async def read_vector_resource(self, unified_resource_id: str) -> JsonValue | RedisFailure: ...
async def write_vector_resource(
self, unified_resource_id: str, record: Mapping[str, object], *, ttl: timedelta | None
) -> None | RedisFailure: ...
async def invalidate_vector_resource(
self, unified_resource_id: str, *, ttl: timedelta | None
) -> None | RedisFailure: ...
@dataclass(frozen=True, slots=True)
class TransportManagedVectorResourceStore:
transport: RedisTransport
async def read_vector_resource(self, unified_resource_id: str) -> JsonValue | RedisFailure:
raw: Final = await self.transport.get(vector_resource_key(unified_resource_id))
if raw is None or isinstance(raw, RedisFailure):
return raw
try:
return _RECORD.validate_python(json.loads(raw.decode("utf-8")))
except (ValueError, UnicodeError):
try:
return _RECORD.validate_python(ast.literal_eval(raw.decode("utf-8")))
except (ValueError, SyntaxError, UnicodeError) as exc:
return RedisFailure(kind="decode", message=str(exc))
async def write_vector_resource(
self, unified_resource_id: str, record: Mapping[str, object], *, ttl: timedelta | None
) -> None | RedisFailure:
try:
encoded: Final = json.dumps(record)
except (TypeError, ValueError) as exc:
return RedisFailure(kind="decode", message=str(exc))
return await self._write(unified_resource_id, encoded, ttl)
async def invalidate_vector_resource(
self, unified_resource_id: str, *, ttl: timedelta | None
) -> None | RedisFailure:
return await self._write(unified_resource_id, "null", ttl)
async def _write(self, unified_resource_id: str, encoded: str, ttl: timedelta | None) -> None | RedisFailure:
if ttl is not None and ttl <= timedelta(0):
return RedisFailure(kind="server", message="invalid expire time in set")
result: Final = await self.transport.set(vector_resource_key(unified_resource_id), encoded, ttl=ttl)
return result if isinstance(result, RedisFailure) else None

View file

@ -10,6 +10,7 @@ import pytest
from litellm.llms.base_llm.managed_resources.base_managed_resource import (
BaseManagedResource,
)
from litellm.caching.dual_cache import DualCache
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
@ -114,9 +115,10 @@ async def test_list_identity_less_caller_returns_empty_without_query():
[("team-eng", True), ("team-sales", False), (None, False)],
)
async def test_can_access_uses_team_id_for_service_account(caller_team_id, expected):
cache = MagicMock()
cache.async_get_cache = AsyncMock(
return_value={
cache = DualCache()
await cache.async_set_cache(
key="rid",
value={
"created_by": None,
"team_id": "team-eng",
}

View file

@ -0,0 +1,224 @@
import base64
import json
import time
from collections.abc import Mapping
from dataclasses import dataclass
from types import SimpleNamespace
from typing import Final, cast
from unittest.mock import patch
import fakeredis
import pytest
from litellm_enterprise.proxy.hooks.managed_vector_stores import PROXY_LiteLLMManagedVectorStores
from redis.exceptions import TimeoutError as RedisTimeoutError
from litellm.caching.dual_cache import DualCache
from litellm.proxy.utils import InternalUsageCache, PrismaClient
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.types.vector_stores import VectorStoreCreateResponse
from tests._support.fake_redis import fake_redis_cache
@dataclass(frozen=True)
class _Row:
data: Mapping[str, object]
@property
def resource_object(self) -> object:
return self.data.get("resource_object")
def model_dump(self) -> dict[str, object]:
return dict(self.data)
class _Table:
def __init__(self, row: _Row | None = None, raw: fakeredis.FakeAsyncRedis | None = None) -> None:
self.row = row
self.reads = 0
self.raw = raw
async def create(self, data: Mapping[str, object]) -> _Row:
self.row = _Row(data)
return self.row
async def find_first(self, where: Mapping[str, object]) -> _Row | None:
self.reads += 1
return self.row if self.row and self.row.data["unified_resource_id"] == where["unified_resource_id"] else None
async def delete(self, where: Mapping[str, object]) -> None:
if self.raw is not None:
assert await self.raw.get(f"tenant:{where['unified_resource_id']}") == b"null"
self.row = None
def _resource() -> VectorStoreCreateResponse:
return cast(VectorStoreCreateResponse, {"id": "vs-é", "object": "vector_store", "name": "文", "usage_bytes": 2**70})
def _hook(cache: DualCache, table: _Table) -> PROXY_LiteLLMManagedVectorStores:
return PROXY_LiteLLMManagedVectorStores(
InternalUsageCache(cache),
cast(PrismaClient, SimpleNamespace(db=SimpleNamespace(litellm_managedvectorstoretable=table))),
)
@pytest.mark.asyncio
@pytest.mark.parametrize("ttl", [1, 12.9, None, 0.5])
async def test_vector_wire_matches_legacy_with_namespace_and_effective_ttl(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, ttl: float | None
) -> None:
cache: Final = fake_redis_cache(server, namespace="tenant")
dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=ttl, default_redis_ttl=999)
table: Final = _Table()
hook: Final = _hook(dual, table)
resource: Final = _resource()
unified_id: Final = hook.generate_unified_resource_id([resource], ["model-a"])
assert (
base64.urlsafe_b64decode(unified_id + "=" * (-len(unified_id) % 4))
.decode()
.startswith("litellm_proxy:vector_store;unified_id,")
)
record: Final = {
"unified_resource_id": unified_id,
"resource_object": resource,
"model_mappings": {"model-a": "vs-é"},
"flat_model_resource_ids": ["vs-é"],
"created_by": "owner",
"team_id": "team-a",
"updated_by": "owner",
"extra": "文",
}
with patch("time.time", return_value=time.time()):
await hook.store_unified_resource_id(
unified_id,
resource,
None,
{"model-a": "vs-é"},
UserAPIKeyAuth(user_id="owner", team_id="team-a"),
{"extra": "文"},
)
await cache.async_set_cache("legacy", record, **({"ttl": ttl} if ttl is not None else {}))
assert await raw.get(f"tenant:{unified_id}") == await raw.get("tenant:legacy")
assert await raw.pttl(f"tenant:{unified_id}") == await raw.pttl("tenant:legacy")
assert await raw.get(f"tenant:{unified_id}") == (None if ttl == 0.5 else json.dumps(record).encode())
assert await cache.async_get_cache(unified_id) == (None if ttl == 0.5 else record)
assert await raw.get(f"tenant:tenant:{unified_id}") is None
assert table.row is not None
assert table.row.data == {
**record,
"resource_object": json.dumps(resource),
"model_mappings": json.dumps(record["model_mappings"]),
}
@pytest.mark.asyncio
async def test_old_vector_writer_new_reader_preserves_access_mapping_and_expiry(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
resource_id: Final = base64.urlsafe_b64encode(
b"litellm_proxy:vector_store;unified_id,uuid;target_model_names,model;resource_id,vs-1;model_id,model"
).decode().rstrip("=")
record: Final = {
"unified_resource_id": resource_id,
"resource_object": _resource(),
"model_mappings": {"model": "vs-é"},
"created_by": "owner",
"team_id": "team-a",
}
with patch("time.time", return_value=1_800_000_000):
cache: Final = fake_redis_cache(server, namespace="tenant")
dual: Final = DualCache(redis_cache=cache, default_in_memory_ttl=1)
table: Final = _Table(_Row({**record, "model_mappings": json.dumps(record["model_mappings"])}))
hook: Final = _hook(dual, table)
await cache.async_set_cache(resource_id, record, ttl=1)
assert await hook.get_unified_resource_id(resource_id) == record
assert await hook.can_user_access_unified_resource_id(resource_id, UserAPIKeyAuth(user_id="owner")) is True
assert await hook.can_user_access_unified_resource_id(resource_id, UserAPIKeyAuth(team_id="team-a")) is True
assert await hook.can_user_access_unified_resource_id(resource_id, UserAPIKeyAuth(team_id="team-b")) is False
assert await hook.get_model_resource_id_mapping([resource_id], None) == {resource_id: {"model": "vs-é"}}
assert table.reads == 0
with patch("time.time", return_value=1_800_000_002):
assert await hook.get_model_resource_id_mapping([resource_id], None) == {resource_id: {"model": "vs-é"}}
assert table.reads == 1
assert await raw.get(f"tenant:{resource_id}") is None
@pytest.mark.asyncio
@pytest.mark.parametrize("encoded", [b"null", b"{}", b"broken", b"\xff"])
async def test_vector_null_or_malformed_value_falls_back_without_backfill(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis, encoded: bytes
) -> None:
row: Final = _Row({"unified_resource_id": "rid", "resource_object": _resource()})
table: Final = _Table(row)
hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server)), table)
await raw.set("rid", encoded)
assert await hook.get_unified_resource_id("rid") == row.model_dump()
assert await raw.get("rid") == encoded
assert table.reads == 1
@pytest.mark.asyncio
async def test_vector_invalidation_precedes_database_delete_and_keeps_null_ttl(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
row: Final = _Row({"unified_resource_id": "rid", "resource_object": _resource()})
table: Final = _Table(row, raw)
hook: Final = _hook(
DualCache(redis_cache=fake_redis_cache(server, namespace="tenant"), default_in_memory_ttl=1), table
)
with patch("time.time", return_value=time.time()):
assert await hook.delete_unified_resource_id("rid") == _resource()
assert await raw.get("tenant:rid") == b"null"
assert await raw.pttl("tenant:rid") == 1000
assert await hook.get_unified_resource_id("rid") is None
assert table.reads == 2
@pytest.mark.asyncio
async def test_vector_outage_keeps_local_and_database_writes(server: fakeredis.FakeServer) -> None:
cache: Final = fake_redis_cache(server)
dual: Final = DualCache(redis_cache=cache)
table: Final = _Table()
hook: Final = _hook(dual, table)
server.connected = False
await hook.store_unified_resource_id("rid", _resource(), None, {"model": "vs-é"}, UserAPIKeyAuth(user_id="owner"))
assert (await hook.get_unified_resource_id("rid"))["resource_object"] == _resource()
assert table.reads == 0
dual.in_memory_cache.flush_cache()
assert table.row is not None
for _ in range(8):
assert await hook.get_unified_resource_id("rid") == table.row.model_dump()
assert cache._circuit_breaker.state == "open"
@pytest.mark.asyncio
async def test_vector_unknown_applied_write_does_not_undo_cache_or_database(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
cache: Final = fake_redis_cache(server)
table: Final = _Table()
hook: Final = _hook(DualCache(redis_cache=cache, default_in_memory_ttl=1), table)
async def applied_then_timeout(name: str, value: str, *, px: int | None, nx: bool) -> bool:
await raw.set(name, value, px=px, nx=nx)
raise RedisTimeoutError("reply lost after apply")
with patch.object(cache.init_async_client(), "set", side_effect=applied_then_timeout):
await hook.store_unified_resource_id(
"rid", _resource(), None, {"model": "vs-é"}, UserAPIKeyAuth(user_id="owner")
)
assert await raw.get("rid") == json.dumps(await hook.get_unified_resource_id("rid")).encode()
assert table.row is not None
@pytest.mark.asyncio
async def test_absent_vector_object_does_not_overwrite_cache(
server: fakeredis.FakeServer, raw: fakeredis.FakeAsyncRedis
) -> None:
table: Final = _Table()
hook: Final = _hook(DualCache(redis_cache=fake_redis_cache(server)), table)
await raw.set("rid", b"previous")
await hook.store_unified_resource_id("rid", None, None, {"model": "vs-é"}, UserAPIKeyAuth())
assert await raw.get("rid") == b"previous"
assert table.row is not None
assert "resource_object" not in table.row.data