mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(redis): move user provider credentials into a typed store
This commit is contained in:
parent
9c8a9923b0
commit
7313deb726
5 changed files with 433 additions and 194 deletions
|
|
@ -7,16 +7,9 @@ payload encrypted; the request-time cache carries only ciphertext plus a
|
|||
negative marker so an unconnected user does not hit the DB per request.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
Protocol,
|
||||
cast, # noqa: TID251 # narrows the untyped Redis cache to the str-keyed Protocol
|
||||
)
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -25,15 +18,19 @@ from litellm.caching.dual_cache import DualCache
|
|||
if TYPE_CHECKING:
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
from litellm.constants import (
|
||||
GITHUB_COPILOT_DEFAULT_HOST,
|
||||
GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS,
|
||||
USER_PROVIDER_CREDENTIAL_CACHE_PREFIX,
|
||||
USER_PROVIDER_CREDENTIAL_NOT_CONNECTED,
|
||||
)
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
||||
decrypt_value_helper,
|
||||
encrypt_value_helper,
|
||||
from litellm.redis.stores.user_provider_credentials import (
|
||||
GithubCopilotUserConnectionPayload,
|
||||
InvalidCredentialValue,
|
||||
TransportUserProviderCredentialStore,
|
||||
UserProviderCredentialStore,
|
||||
decode_credential,
|
||||
encode_credential,
|
||||
user_provider_credential_key,
|
||||
)
|
||||
from litellm.redis.transport.contract import RedisFailure
|
||||
from litellm.repositories.chunked_in import find_many_in
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.table_repositories import PrismaTableRepository
|
||||
|
|
@ -44,14 +41,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
_NOT_CONNECTED: Final = USER_PROVIDER_CREDENTIAL_NOT_CONNECTED
|
||||
|
||||
|
||||
class GithubCopilotUserConnectionPayload(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", frozen=True)
|
||||
|
||||
access_token: str = Field(repr=False)
|
||||
github_login: str
|
||||
github_host: str = GITHUB_COPILOT_DEFAULT_HOST
|
||||
_cache_key: Final = user_provider_credential_key
|
||||
|
||||
|
||||
class _UserProviderCredentialsRepository(PrismaTableRepository["prisma_models.LiteLLM_UserProviderCredentials"]):
|
||||
|
|
@ -64,28 +54,12 @@ def _table(
|
|||
return _UserProviderCredentialsRepository(prisma_client, use_writer=True).table
|
||||
|
||||
|
||||
def _cache_key(user_id: str, credential_name: str) -> str:
|
||||
pair: Final = json.dumps([user_id, credential_name], separators=(",", ":"))
|
||||
return f"{USER_PROVIDER_CREDENTIAL_CACHE_PREFIX}:{pair}"
|
||||
|
||||
|
||||
def _encode(payload: GithubCopilotUserConnectionPayload) -> str:
|
||||
return encrypt_value_helper(payload.model_dump_json())
|
||||
return encode_credential(payload)
|
||||
|
||||
|
||||
def decode_user_provider_credential(stored: str) -> GithubCopilotUserConnectionPayload | None:
|
||||
decrypted: Final = decrypt_value_helper(
|
||||
value=stored,
|
||||
key="user_provider_credential",
|
||||
exception_type="debug",
|
||||
return_original_value=False,
|
||||
)
|
||||
if decrypted is None:
|
||||
return None
|
||||
try:
|
||||
return GithubCopilotUserConnectionPayload.model_validate_json(decrypted)
|
||||
except ValidationError:
|
||||
return None
|
||||
return decode_credential(stored)
|
||||
|
||||
|
||||
async def upsert_user_provider_credential(
|
||||
|
|
@ -158,47 +132,15 @@ async def list_user_provider_credentials_for_credential(
|
|||
return await _table(prisma_client).find_many(where={"credential_name": credential_name})
|
||||
|
||||
|
||||
class _StringKeyCache(Protocol):
|
||||
async def async_get_cache(
|
||||
self,
|
||||
key: str,
|
||||
**kwargs: object, # kwargs-ok: RedisCache accepts optional cache kwargs
|
||||
) -> object: ...
|
||||
|
||||
async def async_set_cache(
|
||||
self,
|
||||
key: str,
|
||||
value: str,
|
||||
**kwargs: object, # kwargs-ok: RedisCache accepts ttl/nx kwargs
|
||||
) -> None: ...
|
||||
|
||||
async def async_delete_cache(
|
||||
self,
|
||||
key: str,
|
||||
**kwargs: object, # kwargs-ok: RedisCache accepts optional cache kwargs
|
||||
) -> None: ...
|
||||
def _store(cache: "RedisCache") -> UserProviderCredentialStore:
|
||||
return TransportUserProviderCredentialStore(cache.transport())
|
||||
|
||||
|
||||
def _string_cache(cache: "RedisCache") -> _StringKeyCache:
|
||||
return cast(_StringKeyCache, cache) # cast-ok: RedisCache exposes the str-keyed cache protocol
|
||||
|
||||
|
||||
async def _try_cache_get(token_cache: "RedisCache", key: str) -> object:
|
||||
"""Redis errors must read as a plain miss: the DB is the source of truth."""
|
||||
try:
|
||||
return await _string_cache(token_cache).async_get_cache(key)
|
||||
except Exception: # noqa: BLE001 # a Redis outage must fall back to the database, never reject a connected user
|
||||
verbose_proxy_logger.warning("aget_user_provider_credentials: Redis get failed; falling back to the database")
|
||||
return None
|
||||
|
||||
|
||||
async def _try_cache_set(token_cache: "RedisCache", key: str, value: str, nx: bool = False) -> None:
|
||||
try:
|
||||
await _string_cache(token_cache).async_set_cache(
|
||||
key, value, nx=nx, ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS
|
||||
)
|
||||
except Exception: # noqa: BLE001 # caching is best-effort; a Redis outage is not worth failing the request
|
||||
verbose_proxy_logger.warning("aget_user_provider_credentials: Redis set failed; skipping the cache write")
|
||||
async def _try_cache_get(
|
||||
token_cache: "RedisCache", user_id: str, credential_name: str
|
||||
) -> str | None | InvalidCredentialValue:
|
||||
result: Final = await _store(token_cache).read(user_id, credential_name)
|
||||
return None if isinstance(result, RedisFailure) else result
|
||||
|
||||
|
||||
@with_service_target("user_provider_connections")
|
||||
|
|
@ -214,23 +156,11 @@ async def invalidate_user_provider_credential_cache(
|
|||
token_cache: Final = cache.redis_cache
|
||||
if token_cache is None:
|
||||
return True
|
||||
try:
|
||||
await _string_cache(token_cache).async_set_cache(
|
||||
_cache_key(user_id, credential_name),
|
||||
_NOT_CONNECTED,
|
||||
ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # caller decides whether a Redis outage aborts the disconnect
|
||||
verbose_proxy_logger.warning("invalidate_user_provider_credential_cache: Redis tombstone failed")
|
||||
return False
|
||||
# async_set_cache swallows client errors internally, so a write that never
|
||||
# landed looks identical to a success. The tombstone is only trusted when the
|
||||
# key reads back as _NOT_CONNECTED; anything else means revoke failed.
|
||||
readback: Final = await _try_cache_get(token_cache, _cache_key(user_id, credential_name))
|
||||
if readback != _NOT_CONNECTED:
|
||||
verbose_proxy_logger.warning("invalidate_user_provider_credential_cache: tombstone not visible after write")
|
||||
return False
|
||||
return True
|
||||
await _store(token_cache).disconnect(
|
||||
user_id, credential_name, ttl=timedelta(seconds=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS)
|
||||
)
|
||||
readback: Final = await _try_cache_get(token_cache, user_id, credential_name)
|
||||
return readback == _NOT_CONNECTED
|
||||
|
||||
|
||||
@with_service_target("user_provider_connections")
|
||||
|
|
@ -244,10 +174,7 @@ async def drop_user_provider_credential_cache(
|
|||
token_cache: Final = cache.redis_cache
|
||||
if token_cache is None:
|
||||
return
|
||||
try:
|
||||
await _string_cache(token_cache).async_delete_cache(_cache_key(user_id, credential_name))
|
||||
except Exception: # noqa: BLE001 # a Redis outage must not fail a connect
|
||||
verbose_proxy_logger.warning("drop_user_provider_credential_cache: Redis delete failed")
|
||||
await _store(token_cache).delete(user_id, credential_name)
|
||||
|
||||
|
||||
@with_service_target("user_provider_connections")
|
||||
|
|
@ -265,32 +192,21 @@ async def set_user_provider_credential_cache(
|
|||
token_cache: Final = cache.redis_cache
|
||||
if token_cache is None:
|
||||
return True
|
||||
key: Final = _cache_key(user_id, credential_name)
|
||||
try:
|
||||
await _string_cache(token_cache).async_set_cache(
|
||||
key,
|
||||
_encode(payload),
|
||||
ttl=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS,
|
||||
)
|
||||
readback: Final = await _try_cache_get(token_cache, key)
|
||||
write: Final = await _store(token_cache).connect(
|
||||
user_id,
|
||||
credential_name,
|
||||
payload,
|
||||
ttl=timedelta(seconds=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS),
|
||||
)
|
||||
if not (isinstance(write, RedisFailure) and write.kind == "decode"):
|
||||
readback: Final = await _try_cache_get(token_cache, user_id, credential_name)
|
||||
if isinstance(readback, str) and decode_user_provider_credential(readback) == payload:
|
||||
return True
|
||||
verbose_proxy_logger.warning(
|
||||
"set_user_provider_credential_cache: Redis overwrite not visible; falling back to delete"
|
||||
)
|
||||
except Exception: # noqa: BLE001 # a Redis outage must not fail a connect
|
||||
verbose_proxy_logger.warning("set_user_provider_credential_cache: Redis set failed; falling back to delete")
|
||||
try:
|
||||
await _string_cache(token_cache).async_delete_cache(key)
|
||||
except Exception: # noqa: BLE001 # a Redis outage must not fail a connect
|
||||
verbose_proxy_logger.warning("set_user_provider_credential_cache: Redis delete failed")
|
||||
after_delete: Final = await _try_cache_get(token_cache, key)
|
||||
await _store(token_cache).delete(user_id, credential_name)
|
||||
after_delete: Final = await _try_cache_get(token_cache, user_id, credential_name)
|
||||
if after_delete is None:
|
||||
return True
|
||||
if isinstance(after_delete, str) and decode_user_provider_credential(after_delete) == payload:
|
||||
return True
|
||||
verbose_proxy_logger.warning("set_user_provider_credential_cache: stale entry survived delete")
|
||||
return False
|
||||
return isinstance(after_delete, str) and decode_user_provider_credential(after_delete) == payload
|
||||
|
||||
|
||||
@with_service_target("user_provider_connections")
|
||||
|
|
@ -313,8 +229,9 @@ async def aget_user_provider_credentials(
|
|||
cached: Final[dict[str, str]] = {} # mutable-ok: accumulates hits and DB reads
|
||||
misses: Final[list[str]] = [] # mutable-ok: accumulates cache misses
|
||||
for name in names:
|
||||
value = await _try_cache_get(token_cache, _cache_key(user_id, name)) if token_cache is not None else None
|
||||
if value == _NOT_CONNECTED:
|
||||
if (
|
||||
value := await _try_cache_get(token_cache, user_id, name) if token_cache is not None else None
|
||||
) == _NOT_CONNECTED:
|
||||
continue
|
||||
if isinstance(value, str) and value:
|
||||
cached[name] = value
|
||||
|
|
@ -340,13 +257,15 @@ async def aget_user_provider_credentials(
|
|||
found: Final = {row.credential_name: row.credential_b64 for row in rows}
|
||||
for name in misses:
|
||||
if token_cache is not None:
|
||||
await _try_cache_set(token_cache, _cache_key(user_id, name), found.get(name, _NOT_CONNECTED), nx=True)
|
||||
await _store(token_cache).fill(
|
||||
user_id, name, found.get(name), ttl=timedelta(seconds=GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS)
|
||||
)
|
||||
if name in found:
|
||||
cached[name] = found[name]
|
||||
return {
|
||||
name: connection
|
||||
name: found_connection
|
||||
for name, ciphertext in cached.items()
|
||||
if (connection := _connection_from_ciphertext(ciphertext)) is not None
|
||||
if (found_connection := _connection_from_ciphertext(ciphertext)) is not None
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
121
litellm/redis/stores/user_provider_credentials.py
Normal file
121
litellm/redis/stores/user_provider_credentials.py
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
import ast
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.constants import (
|
||||
GITHUB_COPILOT_DEFAULT_HOST,
|
||||
USER_PROVIDER_CREDENTIAL_CACHE_PREFIX,
|
||||
USER_PROVIDER_CREDENTIAL_NOT_CONNECTED,
|
||||
)
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper
|
||||
from litellm.redis.transport.contract import RedisFailure, RedisTransport
|
||||
|
||||
_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InvalidCredentialValue:
|
||||
pass
|
||||
|
||||
|
||||
class GithubCopilotUserConnectionPayload(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore", frozen=True)
|
||||
|
||||
access_token: str = Field(repr=False)
|
||||
github_login: str
|
||||
github_host: str = GITHUB_COPILOT_DEFAULT_HOST
|
||||
|
||||
|
||||
def user_provider_credential_key(user_id: str, credential_name: str) -> str:
|
||||
pair: Final = json.dumps([user_id, credential_name], separators=(",", ":"))
|
||||
return f"{USER_PROVIDER_CREDENTIAL_CACHE_PREFIX}:{pair}"
|
||||
|
||||
|
||||
def encode_credential(payload: GithubCopilotUserConnectionPayload) -> str:
|
||||
return encrypt_value_helper(payload.model_dump_json())
|
||||
|
||||
|
||||
def decode_credential(stored: str) -> GithubCopilotUserConnectionPayload | None:
|
||||
decrypted: Final = decrypt_value_helper(
|
||||
value=stored, key="user_provider_credential", exception_type="debug", return_original_value=False
|
||||
)
|
||||
if decrypted is None:
|
||||
return None
|
||||
try:
|
||||
return GithubCopilotUserConnectionPayload.model_validate_json(decrypted)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _decode_reply(reply: bytes | None | RedisFailure) -> str | None | InvalidCredentialValue | RedisFailure:
|
||||
if not isinstance(reply, bytes):
|
||||
return reply
|
||||
try:
|
||||
decoded: Final = reply.decode("utf-8")
|
||||
try:
|
||||
value: Final = _JSON.validate_json(decoded)
|
||||
except ValidationError:
|
||||
legacy: Final = _JSON.validate_python(ast.literal_eval(decoded))
|
||||
return legacy if isinstance(legacy, str) or legacy is None else InvalidCredentialValue()
|
||||
return value if isinstance(value, str) or value is None else InvalidCredentialValue()
|
||||
except (ValueError, SyntaxError):
|
||||
return RedisFailure(kind="decode", message="Invalid user provider credential cache value")
|
||||
|
||||
|
||||
class UserProviderCredentialStore(Protocol):
|
||||
async def read(self, user_id: str, credential_name: str) -> str | None | InvalidCredentialValue | RedisFailure: ...
|
||||
|
||||
async def fill(
|
||||
self, user_id: str, credential_name: str, ciphertext: str | None, *, ttl: timedelta
|
||||
) -> bool | RedisFailure: ...
|
||||
|
||||
async def connect(
|
||||
self, user_id: str, credential_name: str, payload: GithubCopilotUserConnectionPayload, *, ttl: timedelta
|
||||
) -> bool | RedisFailure: ...
|
||||
|
||||
async def disconnect(self, user_id: str, credential_name: str, *, ttl: timedelta) -> bool | RedisFailure: ...
|
||||
|
||||
async def delete(self, user_id: str, credential_name: str) -> int | RedisFailure: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransportUserProviderCredentialStore:
|
||||
transport: RedisTransport
|
||||
|
||||
async def read(self, user_id: str, credential_name: str) -> str | None | InvalidCredentialValue | RedisFailure:
|
||||
return _decode_reply(await self.transport.get(user_provider_credential_key(user_id, credential_name)))
|
||||
|
||||
async def fill(
|
||||
self, user_id: str, credential_name: str, ciphertext: str | None, *, ttl: timedelta
|
||||
) -> bool | RedisFailure:
|
||||
return await self.transport.set(
|
||||
user_provider_credential_key(user_id, credential_name),
|
||||
json.dumps(ciphertext if ciphertext is not None else USER_PROVIDER_CREDENTIAL_NOT_CONNECTED),
|
||||
ttl=ttl,
|
||||
nx=True,
|
||||
)
|
||||
|
||||
async def connect(
|
||||
self, user_id: str, credential_name: str, payload: GithubCopilotUserConnectionPayload, *, ttl: timedelta
|
||||
) -> bool | RedisFailure:
|
||||
try:
|
||||
ciphertext: Final = encode_credential(payload)
|
||||
except Exception: # noqa: BLE001 # the caller handles encryption failures like failed writes
|
||||
return RedisFailure(kind="decode", message="Cannot encode user provider credential")
|
||||
return await self.transport.set(
|
||||
user_provider_credential_key(user_id, credential_name), json.dumps(ciphertext), ttl=ttl
|
||||
)
|
||||
|
||||
async def disconnect(self, user_id: str, credential_name: str, *, ttl: timedelta) -> bool | RedisFailure:
|
||||
return await self.transport.set(
|
||||
user_provider_credential_key(user_id, credential_name),
|
||||
json.dumps(USER_PROVIDER_CREDENTIAL_NOT_CONNECTED),
|
||||
ttl=ttl,
|
||||
)
|
||||
|
||||
async def delete(self, user_id: str, credential_name: str) -> int | RedisFailure:
|
||||
return await self.transport.delete(user_provider_credential_key(user_id, credential_name))
|
||||
|
|
@ -7,6 +7,7 @@ from types import SimpleNamespace
|
|||
from typing import Final, Protocol
|
||||
from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
|
||||
import fakeredis
|
||||
import httpx
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
|
|
@ -26,10 +27,21 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helpe
|
|||
from litellm.proxy.credential_endpoints.endpoints import _decode_flow_handle, get_llm_router
|
||||
from litellm.proxy.proxy_server import app
|
||||
from litellm.types.utils import CredentialItem
|
||||
from tests._support.fake_redis import fake_redis_cache
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
def _auth_with_redis(cache, auth):
|
||||
redis_client: Final = cache.init_async_client()
|
||||
|
||||
async def authenticate():
|
||||
litellm.in_memory_llm_clients_cache.set_cache(key=cache._get_async_client_cache_key(), value=redis_client)
|
||||
return auth()
|
||||
|
||||
return authenticate
|
||||
|
||||
|
||||
def _as_admin():
|
||||
return UserAPIKeyAuth(api_key="test-key", user_role="proxy_admin")
|
||||
|
||||
|
|
@ -1799,17 +1811,8 @@ def test_update_credential_purges_connections_when_github_host_changes(monkeypat
|
|||
proxy_server.prisma_client.db.litellm_userprovidercredentials = table
|
||||
proxy_server.prisma_client.writer_db = proxy_server.prisma_client.db
|
||||
|
||||
cached_values: Final[dict[str, object]] = {} # mutable-ok: fake Redis stores invalidation writes
|
||||
|
||||
async def set_cache(key: str, value: str, **kwargs: object) -> None:
|
||||
cached_values[key] = value
|
||||
|
||||
async def get_cache(key: str, **kwargs: object) -> object:
|
||||
return cached_values.get(key)
|
||||
|
||||
redis_cache: Final = MagicMock()
|
||||
redis_cache.async_set_cache = AsyncMock(side_effect=set_cache)
|
||||
redis_cache.async_get_cache = AsyncMock(side_effect=get_cache)
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis_cache: Final = fake_redis_cache(server)
|
||||
monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache(redis_cache=redis_cache))
|
||||
|
||||
session_key: Final = _session_cache_key("user-a", "copilot", "github.com")
|
||||
|
|
@ -1825,6 +1828,7 @@ def test_update_credential_purges_connections_when_github_host_changes(monkeypat
|
|||
"credential_values": {"github_copilot_host": "octo.ghe.com"},
|
||||
"credential_info": {},
|
||||
},
|
||||
auth=_auth_with_redis(redis_cache, _as_admin),
|
||||
)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
|
|
@ -1835,7 +1839,7 @@ def test_update_credential_purges_connections_when_github_host_changes(monkeypat
|
|||
]
|
||||
)
|
||||
table.delete_many.assert_awaited_once_with(where={"credential_name": "copilot"})
|
||||
assert cached_values[upc._cache_key("user-a", "copilot")] == upc._NOT_CONNECTED
|
||||
assert fakeredis.FakeRedis(server=server).get(upc._cache_key("user-a", "copilot")) == json.dumps(upc._NOT_CONNECTED).encode()
|
||||
assert _SESSION_CACHE.get_cache(session_key) is None
|
||||
|
||||
|
||||
|
|
@ -2901,18 +2905,14 @@ def test_delete_user_connection_fails_closed_when_the_tombstone_write_fails(monk
|
|||
"""Redis attached but its set() raises: the disconnect must not delete the row,
|
||||
or the cached token would keep working until TTL with nothing left to revoke."""
|
||||
table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=[_connection_row()])
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
failing_redis = SimpleNamespace(
|
||||
async_set_cache=AsyncMock(side_effect=ConnectionError("redis down")),
|
||||
async_get_cache=AsyncMock(return_value=None),
|
||||
async_delete_cache=AsyncMock(return_value=None),
|
||||
)
|
||||
server: Final = fakeredis.FakeServer()
|
||||
server.connected = False
|
||||
failing_redis: Final = fake_redis_cache(server)
|
||||
monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache(redis_cache=failing_redis))
|
||||
|
||||
response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user())
|
||||
response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_auth_with_redis(failing_redis, _as_user()))
|
||||
assert response.status_code == 503, response.text
|
||||
table.delete_many.assert_not_awaited()
|
||||
|
||||
|
|
@ -2922,19 +2922,13 @@ def test_delete_user_connection_fails_when_the_tombstone_write_silently_noops(mo
|
|||
without writing still looks successful. The disconnect must verify the
|
||||
tombstone by reading the key back and refuse to delete the row."""
|
||||
table = _patch_user_connection_env(monkeypatch, [_per_user_credential()], rows=[_connection_row()])
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
store: dict = {}
|
||||
silent_redis = SimpleNamespace(
|
||||
async_set_cache=AsyncMock(return_value=None), # reports success, writes nothing
|
||||
async_get_cache=AsyncMock(side_effect=lambda key: store.get(key)),
|
||||
async_delete_cache=AsyncMock(return_value=None),
|
||||
)
|
||||
silent_redis: Final = fake_redis_cache(fakeredis.FakeServer())
|
||||
monkeypatch.setattr(silent_redis.init_async_client(), "set", AsyncMock(return_value=None))
|
||||
monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache(redis_cache=silent_redis))
|
||||
|
||||
response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_as_user())
|
||||
response = _call_as("DELETE", "/credentials/copilot-cred/user_connection", auth=_auth_with_redis(silent_redis, _as_user()))
|
||||
assert response.status_code == 503, response.text
|
||||
table.delete_many.assert_not_awaited()
|
||||
|
||||
|
|
@ -2944,16 +2938,16 @@ def test_user_connection_poll_reports_503_when_the_cache_overwrite_cannot_land(m
|
|||
both the overwrite and the delete must surface as a retryable 503 rather
|
||||
than a false "connected"."""
|
||||
table = _patch_user_connection_env(monkeypatch, [_per_user_credential()])
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
stale_redis = SimpleNamespace(
|
||||
async_set_cache=AsyncMock(return_value=None),
|
||||
async_get_cache=AsyncMock(return_value="stale-value"),
|
||||
async_delete_cache=AsyncMock(return_value=None),
|
||||
)
|
||||
server: Final = fakeredis.FakeServer()
|
||||
raw: Final = fakeredis.FakeRedis(server=server)
|
||||
raw.set('user_provider_credential:["user-a","copilot-cred"]', json.dumps("stale-value"))
|
||||
stale_redis: Final = fake_redis_cache(server)
|
||||
monkeypatch.setattr(stale_redis.init_async_client(), "set", AsyncMock(return_value=None))
|
||||
monkeypatch.setattr(stale_redis.init_async_client(), "delete", AsyncMock(return_value=0))
|
||||
monkeypatch.setattr(proxy_server, "user_api_key_cache", DualCache(redis_cache=stale_redis))
|
||||
redis_auth: Final = _auth_with_redis(stale_redis, _as_user())
|
||||
|
||||
with _github_http(
|
||||
{
|
||||
|
|
@ -2963,12 +2957,12 @@ def test_user_connection_poll_reports_503_when_the_cache_overwrite_cannot_land(m
|
|||
_GITHUB_USER_URL: {"login": "octo"},
|
||||
}
|
||||
):
|
||||
start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=_as_user())
|
||||
start = _call_as("POST", "/credentials/copilot-cred/user_connection/start", auth=redis_auth)
|
||||
poll = _call_as(
|
||||
"POST",
|
||||
"/credentials/copilot-cred/user_connection/poll",
|
||||
json_body={"flow_handle": start.json()["flow_handle"]},
|
||||
auth=_as_user(),
|
||||
auth=redis_auth,
|
||||
)
|
||||
assert poll.status_code == 503, poll.text
|
||||
assert "cache could not be refreshed" in poll.text
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@
|
|||
import json
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import fakeredis
|
||||
import pytest
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
|
|
@ -22,6 +23,7 @@ from litellm.proxy.credential_endpoints.user_provider_credentials import (
|
|||
set_user_provider_credential_cache,
|
||||
upsert_user_provider_credential,
|
||||
)
|
||||
from tests._support.fake_redis import fake_redis_cache
|
||||
|
||||
|
||||
def _prisma(table=None):
|
||||
|
|
@ -41,15 +43,7 @@ def _row(user_id="user-a", credential_name="copilot-cred", credential_b64="ciphe
|
|||
|
||||
|
||||
def _fake_redis(monkeypatch):
|
||||
"""A RedisCache backed by a fakeredis client; two DualCaches can share it."""
|
||||
import fakeredis
|
||||
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
|
||||
fake = fakeredis.FakeAsyncRedis()
|
||||
redis = RedisCache(host="localhost", port=6379)
|
||||
monkeypatch.setattr(redis, "init_async_client", lambda: fake)
|
||||
return redis
|
||||
return fake_redis_cache(fakeredis.FakeServer())
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -279,8 +273,9 @@ async def test_redis_read_failure_falls_back_to_the_database(monkeypatch):
|
|||
"""A broken Redis must not 401 a connected user: reads degrade to a cache miss."""
|
||||
from litellm.proxy.credential_endpoints import user_provider_credentials as upc
|
||||
|
||||
redis = _fake_redis(monkeypatch)
|
||||
monkeypatch.setattr(redis, "async_get_cache", AsyncMock(side_effect=RuntimeError("redis down")))
|
||||
server: Final = fakeredis.FakeServer()
|
||||
server.connected = False
|
||||
redis = fake_redis_cache(server)
|
||||
cache = DualCache(redis_cache=redis)
|
||||
|
||||
payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo")
|
||||
|
|
@ -295,9 +290,9 @@ async def test_redis_read_failure_falls_back_to_the_database(monkeypatch):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redis_write_and_delete_failures_do_not_fail_the_request(monkeypatch):
|
||||
redis = _fake_redis(monkeypatch)
|
||||
monkeypatch.setattr(redis, "async_set_cache", AsyncMock(side_effect=RuntimeError("redis down")))
|
||||
monkeypatch.setattr(redis, "async_delete_cache", AsyncMock(side_effect=RuntimeError("redis down")))
|
||||
server: Final = fakeredis.FakeServer()
|
||||
server.connected = False
|
||||
redis = fake_redis_cache(server)
|
||||
cache = DualCache(redis_cache=redis)
|
||||
|
||||
payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo")
|
||||
|
|
@ -399,15 +394,13 @@ async def test_set_reports_connected_when_delete_clears_a_dropped_write():
|
|||
"""async_set_cache swallows errors: a set that reports success but writes
|
||||
nothing is verified by read-back, and a working delete still resolves the
|
||||
stale entry."""
|
||||
store: dict = {}
|
||||
silent_redis = SimpleNamespace(
|
||||
async_set_cache=AsyncMock(return_value=None),
|
||||
async_get_cache=AsyncMock(side_effect=lambda key: store.get(key)),
|
||||
async_delete_cache=AsyncMock(side_effect=lambda key: store.pop(key, None)),
|
||||
)
|
||||
silent_redis: Final = fake_redis_cache(fakeredis.FakeServer())
|
||||
cache = DualCache(redis_cache=silent_redis)
|
||||
payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo")
|
||||
assert await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload)
|
||||
await silent_redis.async_set_cache('user_provider_credential:["user-a","copilot-cred"]', "stale-value")
|
||||
with patch.object(silent_redis.init_async_client(), "set", AsyncMock(return_value=None)):
|
||||
assert await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload)
|
||||
assert await silent_redis.async_get_cache('user_provider_credential:["user-a","copilot-cred"]') is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -415,15 +408,15 @@ async def test_set_reports_failure_when_a_stale_entry_survives_both_attempts():
|
|||
"""A write silently dropped while a not-connected marker stays cached means
|
||||
the caller cannot trust the connect: the helper reports failure instead of
|
||||
claiming success."""
|
||||
stale = "stale-value"
|
||||
silent_redis = SimpleNamespace(
|
||||
async_set_cache=AsyncMock(return_value=None),
|
||||
async_get_cache=AsyncMock(return_value=stale),
|
||||
async_delete_cache=AsyncMock(return_value=None),
|
||||
)
|
||||
silent_redis: Final = fake_redis_cache(fakeredis.FakeServer())
|
||||
cache = DualCache(redis_cache=silent_redis)
|
||||
payload = GithubCopilotUserConnectionPayload(access_token="gho_secret", github_login="octo")
|
||||
assert not await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload)
|
||||
await silent_redis.async_set_cache('user_provider_credential:["user-a","copilot-cred"]', "stale-value")
|
||||
with (
|
||||
patch.object(silent_redis.init_async_client(), "set", AsyncMock(return_value=None)),
|
||||
patch.object(silent_redis.init_async_client(), "delete", AsyncMock(return_value=0)),
|
||||
):
|
||||
assert not await set_user_provider_credential_cache(cache, "user-a", "copilot-cred", payload)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
212
tests/unit/redis/stores/test_user_provider_credentials.py
Normal file
212
tests/unit/redis/stores/test_user_provider_credentials.py
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
import json
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import fakeredis
|
||||
import pytest
|
||||
from redis.exceptions import ConnectionError, ResponseError
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.constants import GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
|
||||
from litellm.proxy.credential_endpoints.user_provider_credentials import (
|
||||
GithubCopilotUserConnectionPayload,
|
||||
aget_user_provider_credentials,
|
||||
decode_user_provider_credential,
|
||||
drop_user_provider_credential_cache,
|
||||
invalidate_user_provider_credential_cache,
|
||||
set_user_provider_credential_cache,
|
||||
)
|
||||
from litellm.redis.stores.user_provider_credentials import InvalidCredentialValue, TransportUserProviderCredentialStore
|
||||
from litellm.redis.transport.contract import RedisFailure
|
||||
from tests._support.fake_redis import fake_redis_cache
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def encryption_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-test-master")
|
||||
monkeypatch.setattr("time.time", lambda: 1_800_000_000.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_and_new_connections_share_ciphertext_key_tombstones_and_ttl() -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server, namespace="credentials")
|
||||
raw: Final = fakeredis.FakeAsyncRedis(server=server)
|
||||
cache: Final = DualCache(redis_cache=redis)
|
||||
user: Final = 'user:é"'
|
||||
name: Final = "provider:東京"
|
||||
key: Final = "user_provider_credential:" + json.dumps([user, name], separators=(",", ":"))
|
||||
payload: Final = GithubCopilotUserConnectionPayload(access_token="test-token-é", github_login="octo")
|
||||
ciphertext: Final = encrypt_value_helper(payload.model_dump_json())
|
||||
prisma: Final = MagicMock()
|
||||
await redis.async_set_cache(key, ciphertext, ttl=60)
|
||||
assert await raw.get("credentials:" + key) == json.dumps(ciphertext).encode()
|
||||
assert await aget_user_provider_credentials(prisma, cache, user, [name]) == {
|
||||
name: {"access_token": payload.access_token, "github_host": "github.com"}
|
||||
}
|
||||
assert await invalidate_user_provider_credential_cache(cache, user, name)
|
||||
assert await raw.get("credentials:" + key) == b'"__not_connected__"'
|
||||
assert await redis.async_get_cache(key) == "__not_connected__"
|
||||
assert await aget_user_provider_credentials(prisma, cache, user, [name]) == {}
|
||||
assert await set_user_provider_credential_cache(cache, user, name, payload)
|
||||
old_read: Final = await redis.async_get_cache(key)
|
||||
assert decode_user_provider_credential(old_read) == payload
|
||||
assert await raw.get("credentials:" + key) == json.dumps(old_read).encode()
|
||||
assert await raw.pttl("credentials:" + key) == GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS * 1000
|
||||
assert await raw.keys("*") == [("credentials:" + key).encode()]
|
||||
assert await cache.in_memory_cache.async_get_cache(key) is None
|
||||
await drop_user_provider_credential_cache(cache, user, name)
|
||||
assert await raw.get("credentials:" + key) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("applied", [False, True])
|
||||
async def test_ambiguous_tombstone_requires_readback(applied: bool) -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server, namespace="credentials")
|
||||
raw: Final = fakeredis.FakeAsyncRedis(server=server)
|
||||
cache: Final = DualCache(redis_cache=redis)
|
||||
key: Final = 'credentials:user_provider_credential:["u","c"]'
|
||||
|
||||
async def lost_reply(*args: object, **kwargs: object) -> None:
|
||||
if applied:
|
||||
await raw.set(key, b'"__not_connected__"', ex=60)
|
||||
raise ConnectionError("lost reply")
|
||||
|
||||
with patch.object(redis.init_async_client(), "set", lost_reply):
|
||||
assert await invalidate_user_provider_credential_cache(cache, "u", "c") is applied
|
||||
assert await raw.get(key) == (b'"__not_connected__"' if applied else None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ambiguous_connection_write_is_read_before_delete() -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server)
|
||||
raw: Final = fakeredis.FakeAsyncRedis(server=server)
|
||||
payload: Final = GithubCopilotUserConnectionPayload(access_token="test-token", github_login="octo")
|
||||
key: Final = 'user_provider_credential:["u","c"]'
|
||||
ciphertext: Final = encrypt_value_helper(payload.model_dump_json())
|
||||
|
||||
async def lost_reply(*args: object, **kwargs: object) -> None:
|
||||
await raw.set(key, json.dumps(ciphertext), ex=60)
|
||||
raise ConnectionError("lost reply")
|
||||
|
||||
with patch.object(redis.init_async_client(), "set", lost_reply):
|
||||
assert await set_user_provider_credential_cache(DualCache(redis_cache=redis), "u", "c", payload)
|
||||
assert await raw.get(key) == json.dumps(ciphertext).encode()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_outage_uses_database_but_refuses_disconnect_and_keeps_connect_fallback() -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server)
|
||||
cache: Final = DualCache(redis_cache=redis)
|
||||
payload: Final = GithubCopilotUserConnectionPayload(access_token="test-token", github_login="octo")
|
||||
prisma: Final = MagicMock()
|
||||
prisma.writer_db.litellm_userprovidercredentials.find_many = AsyncMock(
|
||||
return_value=[SimpleNamespace(credential_name="c", credential_b64=encrypt_value_helper(payload.model_dump_json()))]
|
||||
)
|
||||
server.connected = False
|
||||
assert await aget_user_provider_credentials(prisma, cache, "u", ["c"]) == {
|
||||
"c": {"access_token": "test-token", "github_host": "github.com"}
|
||||
}
|
||||
assert not await invalidate_user_provider_credential_cache(cache, "u", "c")
|
||||
assert await set_user_provider_credential_cache(cache, "u", "c", payload)
|
||||
store: Final = TransportUserProviderCredentialStore(redis.transport())
|
||||
assert isinstance(await store.read("u", "c"), RedisFailure)
|
||||
redis._circuit_breaker.record_failure()
|
||||
redis._circuit_breaker.record_failure()
|
||||
redis._circuit_breaker.record_failure()
|
||||
result: Final = await store.read("u", "c")
|
||||
assert isinstance(result, RedisFailure) and result.kind == "breaker_open"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("value", [b"garbage", b"\xff"])
|
||||
async def test_malformed_values_are_typed_failures_and_caller_database_misses(value: bytes) -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server)
|
||||
raw: Final = fakeredis.FakeAsyncRedis(server=server)
|
||||
await raw.set('user_provider_credential:["u","c"]', value)
|
||||
store: Final = TransportUserProviderCredentialStore(redis.transport())
|
||||
result: Final = await store.read("u", "c")
|
||||
assert isinstance(result, RedisFailure) and result.kind == "decode"
|
||||
prisma: Final = MagicMock()
|
||||
prisma.writer_db.litellm_userprovidercredentials.find_many = AsyncMock(return_value=[])
|
||||
assert await aget_user_provider_credentials(prisma, DualCache(redis_cache=redis), "u", ["c"]) == {}
|
||||
assert prisma.writer_db.litellm_userprovidercredentials.find_many.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("value", [b"{}", b"42", b"[]", b"false"])
|
||||
async def test_non_string_surviving_failed_write_and_delete_is_not_a_confirmed_connect(value: bytes) -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server)
|
||||
raw: Final = fakeredis.FakeAsyncRedis(server=server)
|
||||
key: Final = 'user_provider_credential:["u","c"]'
|
||||
await raw.set(key, value)
|
||||
store: Final = TransportUserProviderCredentialStore(redis.transport())
|
||||
assert isinstance(await store.read("u", "c"), InvalidCredentialValue)
|
||||
payload: Final = GithubCopilotUserConnectionPayload(access_token="test-token", github_login="octo")
|
||||
with (
|
||||
patch.object(redis.init_async_client(), "set", AsyncMock(side_effect=ResponseError("readonly"))),
|
||||
patch.object(redis.init_async_client(), "delete", AsyncMock(side_effect=ResponseError("readonly"))),
|
||||
):
|
||||
assert not await set_user_provider_credential_cache(DualCache(redis_cache=redis), "u", "c", payload)
|
||||
assert await raw.get(key) == value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tombstone_expires_and_reloads_from_database(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server)
|
||||
cache: Final = DualCache(redis_cache=redis)
|
||||
payload: Final = GithubCopilotUserConnectionPayload(access_token="test-token", github_login="octo")
|
||||
prisma: Final = MagicMock()
|
||||
prisma.writer_db.litellm_userprovidercredentials.find_many = AsyncMock(
|
||||
return_value=[SimpleNamespace(credential_name="c", credential_b64=encrypt_value_helper(payload.model_dump_json()))]
|
||||
)
|
||||
assert await invalidate_user_provider_credential_cache(cache, "u", "c")
|
||||
assert await aget_user_provider_credentials(prisma, cache, "u", ["c"]) == {}
|
||||
monkeypatch.setattr("time.time", lambda: 1_800_000_061.0)
|
||||
assert await aget_user_provider_credentials(prisma, cache, "u", ["c"]) == {
|
||||
"c": {"access_token": "test-token", "github_host": "github.com"}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fill_nx_retains_tombstone_and_server_errors_are_values() -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server)
|
||||
cache: Final = DualCache(redis_cache=redis)
|
||||
store: Final = TransportUserProviderCredentialStore(redis.transport())
|
||||
assert await invalidate_user_provider_credential_cache(cache, "u", "c")
|
||||
assert await store.fill("u", "c", "stale-ciphertext", ttl=timedelta(seconds=60)) is False
|
||||
assert await store.read("u", "c") == "__not_connected__"
|
||||
with patch.object(redis.init_async_client(), "set", AsyncMock(side_effect=ResponseError("readonly"))):
|
||||
result: Final = await store.disconnect("u", "c", ttl=timedelta(seconds=60))
|
||||
assert isinstance(result, RedisFailure) and result.kind == "server"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_in_flight_database_fill_cannot_resurrect_peer_disconnect() -> None:
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis: Final = fake_redis_cache(server)
|
||||
reader: Final = DualCache(redis_cache=redis)
|
||||
peer: Final = DualCache(redis_cache=redis)
|
||||
payload: Final = GithubCopilotUserConnectionPayload(access_token="stale-token", github_login="octo")
|
||||
|
||||
async def database_read(**kwargs: object) -> list[SimpleNamespace]:
|
||||
assert await invalidate_user_provider_credential_cache(peer, "u", "c")
|
||||
return [SimpleNamespace(credential_name="c", credential_b64=encrypt_value_helper(payload.model_dump_json()))]
|
||||
|
||||
prisma: Final = MagicMock()
|
||||
prisma.writer_db.litellm_userprovidercredentials.find_many = database_read
|
||||
assert await aget_user_provider_credentials(prisma, reader, "u", ["c"]) == {
|
||||
"c": {"access_token": "stale-token", "github_host": "github.com"}
|
||||
}
|
||||
assert await aget_user_provider_credentials(prisma, reader, "u", ["c"]) == {}
|
||||
assert await fakeredis.FakeAsyncRedis(server=server).get('user_provider_credential:["u","c"]') == b'"__not_connected__"'
|
||||
Loading…
Add table
Reference in a new issue