mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 1e6d4d265b into c53d7cdd68
This commit is contained in:
commit
353acb0547
7 changed files with 357 additions and 269 deletions
|
|
@ -44,7 +44,7 @@ import secrets
|
|||
from base64 import urlsafe_b64encode
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, Protocol, TypeVar
|
||||
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
|
||||
|
|
@ -98,6 +98,17 @@ from litellm.proxy.mcp_server.outbound_credentials.session_token import (
|
|||
open_session_refresh_token,
|
||||
open_session_token,
|
||||
)
|
||||
from litellm.redis.stores.mcp_single_use import (
|
||||
AssertionRedeemed,
|
||||
AuthCodeUsed,
|
||||
ConnectFlowUsed,
|
||||
RefreshFamilyRevoked,
|
||||
RefreshTokenUsed,
|
||||
SingleUseMarker,
|
||||
TransportMcpSingleUseStore,
|
||||
marker_key,
|
||||
)
|
||||
from litellm.redis.transport.contract import RedisFailure
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
|
|
@ -130,11 +141,7 @@ the 10-minute ceiling RFC 6749 section 4.1.2 recommends. Single-use and PKCE bin
|
|||
unchanged, so the longer window only extends how long the legitimate holder has to paste
|
||||
it, not what an observer could do with it."""
|
||||
_CLAIM_TTL_BUFFER_SECONDS: Final = 60
|
||||
_USED_CODE_CACHE_PREFIX: Final = "mcp_gateway_dcr_code_used:"
|
||||
_USED_FLOW_CACHE_PREFIX: Final = "mcp_gateway_dcr_flow_used:"
|
||||
_USED_REFRESH_CACHE_PREFIX: Final = "mcp_gateway_dcr_refresh_used:"
|
||||
_REFRESH_REPLAYED_DESCRIPTION: Final = "the refresh token was already used"
|
||||
_REVOKED_REFRESH_FAMILY_CACHE_PREFIX: Final = "mcp_gateway_dcr_refresh_family_revoked:"
|
||||
_REFRESH_FAMILY_REVOKED_DESCRIPTION: Final = "the refresh token was revoked"
|
||||
|
||||
MAX_REDIRECT_URIS: Final = 4
|
||||
|
|
@ -260,7 +267,7 @@ class AssertionIdentity(SubjectIdentity):
|
|||
marker must outlive it, the ``scope`` the credential is issued for, and the route groups that
|
||||
scope narrows the credential to (``None`` leaves it the full ``lite login`` credential)."""
|
||||
|
||||
replay_key: str = Field(min_length=1)
|
||||
replay: AssertionRedeemed
|
||||
replay_ttl_seconds: int = Field(gt=0)
|
||||
scope: str | None = None
|
||||
allowed_routes: tuple[str, ...] | None = None
|
||||
|
|
@ -470,14 +477,6 @@ def resolve_session_signing(master_key: str | None, caller: str) -> SessionSigni
|
|||
return SessionSigning(keys=keys, now=datetime.now(timezone.utc))
|
||||
|
||||
|
||||
def _refresh_claim_key(jti: str) -> str:
|
||||
return f"{_USED_REFRESH_CACHE_PREFIX}{jti}"
|
||||
|
||||
|
||||
def _refresh_family_key(family: str) -> str:
|
||||
return f"{_REVOKED_REFRESH_FAMILY_CACHE_PREFIX}{family}"
|
||||
|
||||
|
||||
_REFRESH_CLAIM_TTL_SECONDS: Final = SESSION_REFRESH_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
|
||||
|
||||
|
||||
|
|
@ -1229,7 +1228,7 @@ async def complete_connect_flow(
|
|||
return _oauth_error(400, "invalid_request", "authorize the requested MCP server before finishing")
|
||||
flow_refusal: Final = _claim_refusal(
|
||||
await _SingleUseGuard(cache).claim(
|
||||
f"{_USED_FLOW_CACHE_PREFIX}{opened.jti}", CONNECT_FLOW_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
|
||||
ConnectFlowUsed(opened.jti), CONNECT_FLOW_TTL_SECONDS + _CLAIM_TTL_BUFFER_SECONDS
|
||||
),
|
||||
replayed=_oauth_error(
|
||||
400, "invalid_request", "this connect flow was already completed; restart the connection"
|
||||
|
|
@ -1348,7 +1347,7 @@ class _SingleUseGuard:
|
|||
id replays through a different worker):
|
||||
|
||||
- When a Redis backend is configured it is the SOLE authority: the claim goes straight to Redis
|
||||
(``INCR`` is atomic across replicas), and any Redis fault fails the claim CLOSED — it never falls
|
||||
(``INCRBYFLOAT`` is atomic across replicas), and any Redis fault fails the claim CLOSED — it never falls
|
||||
back to the per-worker in-memory count (``DualCache.async_increment_cache`` does fall back, which
|
||||
is exactly the replay window this avoids).
|
||||
- With no Redis configured (single-replica) the in-memory increment is authoritative within the one
|
||||
|
|
@ -1361,8 +1360,8 @@ class _SingleUseGuard:
|
|||
self._cache = cache
|
||||
|
||||
@with_service_target(_DCR_CLAIMS_TARGET)
|
||||
async def claim(self, key: str, ttl_seconds: int) -> ClaimOutcome:
|
||||
"""Atomically claim ``key``. ``"first"`` iff this caller is the first (increment to 1),
|
||||
async def claim(self, marker: SingleUseMarker, ttl_seconds: int) -> ClaimOutcome:
|
||||
"""Atomically claim ``marker``. ``"first"`` iff this caller is the first (increment to 1),
|
||||
``"replayed"`` on a replay (>1), and ``"unavailable"`` when the claim could not be recorded in
|
||||
the shared backend, which every caller treats as a refusal (fail closed)."""
|
||||
from litellm.proxy.proxy_server import redis_usage_cache # noqa: PLC0415 # circular import at module load
|
||||
|
|
@ -1374,41 +1373,36 @@ class _SingleUseGuard:
|
|||
# proxy already treats as cross-worker, so no call site can wire the guarantee away.
|
||||
redis_cache: Final = redis_usage_cache or getattr(self._cache, "redis_cache", None)
|
||||
if redis_cache is not None:
|
||||
# Shared, atomic authority for multi-replica deployments. Claim ONLY against Redis and fail
|
||||
# CLOSED on any Redis fault (async_increment re-raises) rather than fall back to the
|
||||
# per-worker in-memory count, which would let each replica observe count==1 and replay the id.
|
||||
try:
|
||||
count = await redis_cache.async_increment(key, 1, ttl=ttl_seconds)
|
||||
except Exception as e: # noqa: BLE001 # ANY Redis fault fails the single-use claim closed
|
||||
verbose_logger.warning(
|
||||
"mcp gateway single-use claim: shared cache backend unavailable, failing closed: %s", e
|
||||
)
|
||||
return "unavailable"
|
||||
return "first" if count == 1 else "replayed"
|
||||
match await TransportMcpSingleUseStore(redis_cache.transport()).claim(
|
||||
marker, ttl=timedelta(seconds=ttl_seconds)
|
||||
):
|
||||
case RedisFailure():
|
||||
return "unavailable"
|
||||
case 1:
|
||||
return "first"
|
||||
case _:
|
||||
return "replayed"
|
||||
# No shared backend configured (single-replica): the in-memory increment is authoritative.
|
||||
count = await self._cache.async_increment_cache(key, 1, ttl=ttl_seconds, local_only=True)
|
||||
count: Final = await self._cache.async_increment_cache(marker_key(marker), 1, ttl=ttl_seconds, local_only=True)
|
||||
return "first" if count == 1 else "replayed"
|
||||
|
||||
@with_service_target(_DCR_CLAIMS_TARGET)
|
||||
async def peek(self, key: str) -> Literal["unclaimed", "claimed", "unavailable"]:
|
||||
async def peek(self, marker: SingleUseMarker) -> Literal["unclaimed", "claimed", "unavailable"]:
|
||||
"""Read-only view of a single-use marker, resolved against the same shared authority as
|
||||
:meth:`claim` so introspection observes exactly the record redemption and revocation wrote.
|
||||
A backend fault is ``"unavailable"`` (fail closed) rather than a guess either way, which is why
|
||||
the read goes to the Redis client itself: the cache wrapper's ``async_get_cache`` turns a fault
|
||||
into ``None``, and ``None`` here would pass as unclaimed. The key is namespaced exactly as the
|
||||
wrapper namespaces every write, or a ``namespace`` configured under ``coordination_redis`` or
|
||||
``cache_params`` would hide the marker."""
|
||||
A backend fault is ``"unavailable"`` (fail closed) rather than a guess either way."""
|
||||
from litellm.proxy.proxy_server import redis_usage_cache # noqa: PLC0415 # circular import at module load
|
||||
|
||||
redis_cache: Final = redis_usage_cache or getattr(self._cache, "redis_cache", None)
|
||||
if redis_cache is not None:
|
||||
try:
|
||||
value = await redis_cache.init_async_client().get(redis_cache.check_and_fix_namespace(key=key))
|
||||
except Exception as e: # noqa: BLE001 # ANY Redis fault fails the read closed
|
||||
verbose_logger.warning("mcp gateway single-use peek: shared cache backend unavailable: %s", e)
|
||||
return "unavailable"
|
||||
return "unclaimed" if value is None else "claimed"
|
||||
local: Final = await self._cache.async_get_cache(key, local_only=True)
|
||||
match await TransportMcpSingleUseStore(redis_cache.transport()).peek(marker):
|
||||
case RedisFailure():
|
||||
return "unavailable"
|
||||
case True:
|
||||
return "claimed"
|
||||
case False:
|
||||
return "unclaimed"
|
||||
local: Final = await self._cache.async_get_cache(marker_key(marker), local_only=True)
|
||||
return "unclaimed" if local is None else "claimed"
|
||||
|
||||
|
||||
|
|
@ -1417,7 +1411,7 @@ async def _family_refusal(guard: _SingleUseGuard, family: str | None) -> Respons
|
|||
minted; a backend that cannot say answers 503."""
|
||||
if family is None:
|
||||
return None
|
||||
peeked: Final = await guard.peek(_refresh_family_key(family))
|
||||
peeked: Final = await guard.peek(RefreshFamilyRevoked(family))
|
||||
match peeked:
|
||||
case "unclaimed":
|
||||
return None
|
||||
|
|
@ -1666,7 +1660,7 @@ class _ProxyCredentialIssuer:
|
|||
async def __call__(
|
||||
self,
|
||||
principal: SessionPrincipal,
|
||||
claim_key: str,
|
||||
claim: SingleUseMarker,
|
||||
claim_ttl_seconds: int,
|
||||
replayed: str,
|
||||
family: str | None = None,
|
||||
|
|
@ -1678,7 +1672,7 @@ class _ProxyCredentialIssuer:
|
|||
if not isinstance(minted, MintedProxyCredential):
|
||||
return _mint_failure_response(minted)
|
||||
refusal: Final = _claim_refusal(
|
||||
await self._guard.claim(claim_key, claim_ttl_seconds),
|
||||
await self._guard.claim(claim, claim_ttl_seconds),
|
||||
replayed=_oauth_error(400, "invalid_grant", replayed),
|
||||
)
|
||||
if refusal is not None:
|
||||
|
|
@ -1712,21 +1706,26 @@ class _GrantIssuer:
|
|||
async def __call__(
|
||||
self,
|
||||
principal: SessionPrincipal,
|
||||
claim_key: str,
|
||||
claim: SingleUseMarker,
|
||||
claim_ttl_seconds: int,
|
||||
replayed: str,
|
||||
family: str | None = None,
|
||||
) -> Response:
|
||||
match principal.audience:
|
||||
case None:
|
||||
return await self._issue_session_pair(principal, claim_key, claim_ttl_seconds, replayed, family)
|
||||
return await self._issue_session_pair(principal, claim, claim_ttl_seconds, replayed, family)
|
||||
case "proxy_api":
|
||||
return await self._issue_proxy_credential(principal, claim_key, claim_ttl_seconds, replayed, family)
|
||||
return await self._issue_proxy_credential(principal, claim, claim_ttl_seconds, replayed, family)
|
||||
case _:
|
||||
assert_never(principal.audience)
|
||||
|
||||
async def _issue_session_pair(
|
||||
self, principal: SessionPrincipal, claim_key: str, claim_ttl_seconds: int, replayed: str, family: str | None
|
||||
self,
|
||||
principal: SessionPrincipal,
|
||||
claim: SingleUseMarker,
|
||||
claim_ttl_seconds: int,
|
||||
replayed: str,
|
||||
family: str | None,
|
||||
) -> Response:
|
||||
ended: Final = await _family_refusal(self._guard, family)
|
||||
if ended is not None:
|
||||
|
|
@ -1735,7 +1734,7 @@ class _GrantIssuer:
|
|||
if failure is not None:
|
||||
return _reload_failure_response(failure)
|
||||
refusal: Final = _claim_refusal(
|
||||
await self._guard.claim(claim_key, claim_ttl_seconds),
|
||||
await self._guard.claim(claim, claim_ttl_seconds),
|
||||
replayed=_oauth_error(400, "invalid_grant", replayed),
|
||||
)
|
||||
if refusal is not None:
|
||||
|
|
@ -1743,12 +1742,17 @@ class _GrantIssuer:
|
|||
return _session_token_pair(principal, self._signing.keys, self._signing.now, family)
|
||||
|
||||
async def _issue_proxy_credential(
|
||||
self, principal: SessionPrincipal, claim_key: str, claim_ttl_seconds: int, replayed: str, family: str | None
|
||||
self,
|
||||
principal: SessionPrincipal,
|
||||
claim: SingleUseMarker,
|
||||
claim_ttl_seconds: int,
|
||||
replayed: str,
|
||||
family: str | None,
|
||||
) -> Response:
|
||||
target_refusal: Final = self._proxy_api_target_refusal()
|
||||
if target_refusal is not None:
|
||||
return target_refusal
|
||||
return await self._proxy_credential(principal, claim_key, claim_ttl_seconds, replayed, family)
|
||||
return await self._proxy_credential(principal, claim, claim_ttl_seconds, replayed, family)
|
||||
|
||||
async def exchange(
|
||||
self, subject_token: str, client_id: str, exchange_subject_token: ExchangeSubjectToken
|
||||
|
|
@ -1787,7 +1791,7 @@ class _GrantIssuer:
|
|||
if isinstance(identity, SubjectTokenRefusal):
|
||||
return _subject_token_refusal_response(identity)
|
||||
refused: Final = _claim_refusal(
|
||||
await self._guard.claim(identity.replay_key, identity.replay_ttl_seconds),
|
||||
await self._guard.claim(identity.replay, identity.replay_ttl_seconds),
|
||||
_oauth_error(400, "invalid_grant", "the assertion has already been redeemed"),
|
||||
)
|
||||
if refused is not None:
|
||||
|
|
@ -1853,7 +1857,7 @@ async def _authorization_code_grant(
|
|||
audience=parsed.audience,
|
||||
team_id=parsed.team_id,
|
||||
),
|
||||
claim_key=f"{_USED_CODE_CACHE_PREFIX}{parsed.jti}",
|
||||
claim=AuthCodeUsed(parsed.jti),
|
||||
claim_ttl_seconds=parsed.exp - int(now.timestamp()) + _CLAIM_TTL_BUFFER_SECONDS,
|
||||
replayed="the authorization code was already used",
|
||||
)
|
||||
|
|
@ -1891,7 +1895,7 @@ async def _refresh_token_grant(
|
|||
# credential file, and the chain it recovers into has to be alive. Only a revocation ends it.
|
||||
return await issue(
|
||||
opened.principal,
|
||||
claim_key=_refresh_claim_key(opened.jti),
|
||||
claim=RefreshTokenUsed(opened.jti),
|
||||
claim_ttl_seconds=_REFRESH_CLAIM_TTL_SECONDS,
|
||||
replayed=_REFRESH_REPLAYED_DESCRIPTION,
|
||||
family=opened.family,
|
||||
|
|
@ -1922,7 +1926,7 @@ async def refresh_proxy_credential(
|
|||
issue: Final = _ProxyCredentialIssuer(signing, mint_proxy_credential, _SingleUseGuard(cache))
|
||||
return await issue(
|
||||
opened.principal,
|
||||
claim_key=_refresh_claim_key(opened.jti),
|
||||
claim=RefreshTokenUsed(opened.jti),
|
||||
claim_ttl_seconds=_REFRESH_CLAIM_TTL_SECONDS,
|
||||
replayed=_REFRESH_REPLAYED_DESCRIPTION,
|
||||
family=opened.family,
|
||||
|
|
@ -1999,8 +2003,8 @@ async def revoke_session_refresh_token(
|
|||
opened: Final = open_session_refresh_bearer(token, signing.keys, signing.now, expected_client_id=client_id)
|
||||
if isinstance(opened, SessionRefreshOpened):
|
||||
guard: Final = _SingleUseGuard(cache)
|
||||
ended: Final = await guard.claim(_refresh_family_key(opened.family), _REFRESH_CLAIM_TTL_SECONDS)
|
||||
burned: Final = await guard.claim(_refresh_claim_key(opened.jti), _REFRESH_CLAIM_TTL_SECONDS)
|
||||
ended: Final = await guard.claim(RefreshFamilyRevoked(opened.family), _REFRESH_CLAIM_TTL_SECONDS)
|
||||
burned: Final = await guard.claim(RefreshTokenUsed(opened.jti), _REFRESH_CLAIM_TTL_SECONDS)
|
||||
if "unavailable" in (burned, ended):
|
||||
return _oauth_error(503, "temporarily_unavailable", _CLAIM_UNAVAILABLE_DESCRIPTION)
|
||||
return Response(content="{}", media_type="application/json", headers=TOKEN_NO_CACHE_HEADERS)
|
||||
|
|
@ -2085,8 +2089,8 @@ async def introspect_gateway_token(
|
|||
return _inactive_introspection_response()
|
||||
if opened.kind == "session_refresh":
|
||||
guard: Final = _SingleUseGuard(cache)
|
||||
used: Final = await guard.peek(_refresh_claim_key(opened.jti))
|
||||
ended: Final = await guard.peek(_refresh_family_key(opened.family))
|
||||
used: Final = await guard.peek(RefreshTokenUsed(opened.jti))
|
||||
ended: Final = await guard.peek(RefreshFamilyRevoked(opened.family))
|
||||
if "unavailable" in (used, ended):
|
||||
return _oauth_error(503, "temporarily_unavailable", _CLAIM_UNAVAILABLE_DESCRIPTION)
|
||||
if "claimed" in (used, ended):
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ from __future__ import annotations
|
|||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import hmac
|
||||
import math
|
||||
from collections.abc import Awaitable, Iterator, Mapping, Sequence
|
||||
|
|
@ -41,12 +40,12 @@ from litellm.proxy.mcp_server.idp_token_exchange import (
|
|||
read_token_exchange_prerequisites,
|
||||
)
|
||||
from litellm.proxy.mcp_server.oauth_utils import proxy_base_url_from_env
|
||||
from litellm.redis.stores.mcp_single_use import AssertionRedeemed
|
||||
|
||||
ID_JAG_REQUIRED_CLAIMS: Final = ("iss", "sub", "aud", "exp", "iat", "jti", "client_id")
|
||||
REJECTED_ASSERTION: Final = "assertion was rejected"
|
||||
TEAM_HEADER: Final = "x-litellm-team-id"
|
||||
_INACTIVE_SUBJECT: Final = frozenset(("no_active_key", "unresolvable"))
|
||||
_REPLAY_KEY_PREFIX: Final = "mcp_gateway_dcr:id_jag:"
|
||||
SCOPE_MCP: Final = "litellm:mcp"
|
||||
SCOPE_LLM: Final = "litellm:llm"
|
||||
_RESOURCE_CLAIM_ADAPTER: Final = TypeAdapter(list[str])
|
||||
|
|
@ -479,7 +478,7 @@ async def identity_from_id_jag(
|
|||
marker: Final = _replay_marker(verified, redemption)
|
||||
if isinstance(marker, SubjectTokenRefusal):
|
||||
return marker
|
||||
replay_key, replay_ttl_seconds = marker
|
||||
replay, replay_ttl_seconds = marker
|
||||
mcp_resource: Final = resource_names_mcp(verified, prerequisites.base_url)
|
||||
if isinstance(mcp_resource, SubjectTokenRefusal):
|
||||
return mcp_resource
|
||||
|
|
@ -502,7 +501,7 @@ async def identity_from_id_jag(
|
|||
return AssertionIdentity(
|
||||
user_id=subject.user_id,
|
||||
team_id=subject.team_id,
|
||||
replay_key=replay_key,
|
||||
replay=replay,
|
||||
replay_ttl_seconds=replay_ttl_seconds,
|
||||
scope=scope.scope,
|
||||
allowed_routes=scope.allowed_routes,
|
||||
|
|
@ -511,8 +510,8 @@ async def identity_from_id_jag(
|
|||
|
||||
def _replay_marker(
|
||||
verified: Mapping[str, object], redemption: AssertionRedemption
|
||||
) -> tuple[str, int] | SubjectTokenRefusal:
|
||||
"""The single-use key and its lifetime for a verified assertion, once the assertion is bound to
|
||||
) -> tuple[AssertionRedeemed, int] | SubjectTokenRefusal:
|
||||
"""The single-use marker and its lifetime for a verified assertion, once the assertion is bound to
|
||||
the presenting app, carries no proof-of-possession the gateway cannot check, and is still alive."""
|
||||
if verified.get("client_id") != redemption.client_id:
|
||||
return _rejected("the assertion's client_id is not the requesting app that presented it")
|
||||
|
|
@ -526,7 +525,7 @@ def _replay_marker(
|
|||
replay_ttl_seconds: Final = math.ceil(exp + redemption.leeway_seconds - redemption.now)
|
||||
if replay_ttl_seconds <= 0:
|
||||
return _rejected("the assertion has expired")
|
||||
return _REPLAY_KEY_PREFIX + hashlib.sha256(f"{issuer}\n{jti}".encode()).hexdigest(), replay_ttl_seconds
|
||||
return AssertionRedeemed(issuer=issuer, jti=jti), replay_ttl_seconds
|
||||
|
||||
|
||||
def _rejected(reason: str) -> SubjectTokenRefusal:
|
||||
|
|
|
|||
91
litellm/redis/stores/mcp_single_use.py
Normal file
91
litellm/redis/stores/mcp_single_use.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import hashlib
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
from typing import Final, Protocol, TypeAlias
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.redis.transport.contract import EvalReply, RedisFailure, RedisTransport
|
||||
|
||||
_CLAIM: Final = """
|
||||
local count = redis.call("incrbyfloat", KEYS[1], ARGV[1])
|
||||
if redis.call("ttl", KEYS[1]) < 0 then
|
||||
redis.call("expire", KEYS[1], ARGV[2])
|
||||
end
|
||||
return count
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConnectFlowUsed:
|
||||
jti: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AuthCodeUsed:
|
||||
jti: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RefreshTokenUsed:
|
||||
jti: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RefreshFamilyRevoked:
|
||||
family: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AssertionRedeemed:
|
||||
issuer: str
|
||||
jti: str
|
||||
|
||||
|
||||
SingleUseMarker: TypeAlias = (
|
||||
ConnectFlowUsed | AuthCodeUsed | RefreshTokenUsed | RefreshFamilyRevoked | AssertionRedeemed
|
||||
)
|
||||
|
||||
|
||||
def marker_key(marker: SingleUseMarker) -> str:
|
||||
match marker:
|
||||
case ConnectFlowUsed(jti=jti):
|
||||
return f"mcp_gateway_dcr_flow_used:{jti}"
|
||||
case AuthCodeUsed(jti=jti):
|
||||
return f"mcp_gateway_dcr_code_used:{jti}"
|
||||
case RefreshTokenUsed(jti=jti):
|
||||
return f"mcp_gateway_dcr_refresh_used:{jti}"
|
||||
case RefreshFamilyRevoked(family=family):
|
||||
return f"mcp_gateway_dcr_refresh_family_revoked:{family}"
|
||||
case AssertionRedeemed(issuer=issuer, jti=jti):
|
||||
return "mcp_gateway_dcr:id_jag:" + hashlib.sha256(f"{issuer}\n{jti}".encode()).hexdigest()
|
||||
case _:
|
||||
assert_never(marker)
|
||||
|
||||
|
||||
class McpSingleUseStore(Protocol):
|
||||
async def claim(self, marker: SingleUseMarker, *, ttl: timedelta) -> int | RedisFailure: ...
|
||||
|
||||
async def peek(self, marker: SingleUseMarker) -> bool | RedisFailure: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransportMcpSingleUseStore:
|
||||
transport: RedisTransport
|
||||
|
||||
async def claim(self, marker: SingleUseMarker, *, ttl: timedelta) -> int | RedisFailure:
|
||||
reply: Final = await self.transport.eval(_CLAIM, (marker_key(marker),), (1, ttl // timedelta(seconds=1)))
|
||||
return reply if isinstance(reply, RedisFailure) else _count(reply)
|
||||
|
||||
async def peek(self, marker: SingleUseMarker) -> bool | RedisFailure:
|
||||
reply: Final = await self.transport.get(marker_key(marker))
|
||||
return reply if isinstance(reply, RedisFailure) else reply is not None
|
||||
|
||||
|
||||
def _count(reply: EvalReply) -> int | RedisFailure:
|
||||
if not isinstance(reply, bytes):
|
||||
return RedisFailure(kind="decode", message=f"single-use count is not a bulk string: {reply!r}")
|
||||
try:
|
||||
return int(float(reply))
|
||||
except ValueError:
|
||||
return RedisFailure(kind="decode", message=f"single-use count is not a number: {reply!r}")
|
||||
|
|
@ -14,12 +14,14 @@ from types import MappingProxyType
|
|||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import fakeredis
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.anthropic_endpoints import gateway_endpoints
|
||||
from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken
|
||||
|
|
@ -72,55 +74,18 @@ _COMPLETED_SESSION: Final = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
class _SharedRedisFake:
|
||||
"""A Redis double with a configured namespace, prefixing every key its wrapper methods touch the
|
||||
way ``RedisCache`` does, while its raw client answers only the exact key it is asked for."""
|
||||
|
||||
namespace: Final = "qa"
|
||||
|
||||
class _SharedRedisCache(RedisCache):
|
||||
def __init__(self) -> None:
|
||||
self.values: Mapping[str, object] = MappingProxyType({})
|
||||
self.counters: Mapping[str, float] = MappingProxyType({})
|
||||
self._server: Final = fakeredis.FakeServer()
|
||||
super().__init__(host="127.0.0.1", port=1, namespace="qa", socket_timeout=0.1)
|
||||
self.redis_client = fakeredis.FakeRedis(server=self._server)
|
||||
|
||||
def check_and_fix_namespace(self, key: str) -> str:
|
||||
return key if key.startswith(f"{self.namespace}:") else f"{self.namespace}:{key}"
|
||||
|
||||
def set_cache(self, key: str, value: object, **kwargs: object) -> None:
|
||||
self.values = MappingProxyType({**self.values, self.check_and_fix_namespace(key): value})
|
||||
|
||||
def get_cache(self, key: str, **kwargs: object) -> object:
|
||||
return self.values.get(self.check_and_fix_namespace(key))
|
||||
|
||||
def delete_cache(self, key: str) -> None:
|
||||
gone: Final = self.check_and_fix_namespace(key)
|
||||
self.values = MappingProxyType({name: value for name, value in self.values.items() if name != gone})
|
||||
|
||||
async def async_delete_cache(self, key: str) -> None:
|
||||
self.delete_cache(key)
|
||||
|
||||
async def async_increment(self, key: str, value: float, **kwargs: object) -> float:
|
||||
namespaced: Final = self.check_and_fix_namespace(key)
|
||||
incremented: Final = self.counters.get(namespaced, 0) + value
|
||||
self.counters = MappingProxyType({**self.counters, namespaced: incremented})
|
||||
return incremented
|
||||
|
||||
async def async_get_cache(self, key: str, **kwargs: object) -> object:
|
||||
return self.values.get(self.check_and_fix_namespace(key))
|
||||
|
||||
def init_async_client(self) -> "_SharedRedisClientFake":
|
||||
return _SharedRedisClientFake(self)
|
||||
def init_async_client(self) -> fakeredis.FakeAsyncRedis:
|
||||
return fakeredis.FakeAsyncRedis(server=self._server)
|
||||
|
||||
|
||||
class _SharedRedisClientFake:
|
||||
def __init__(self, store: _SharedRedisFake) -> None:
|
||||
self._store = store
|
||||
|
||||
async def get(self, key: str) -> object:
|
||||
return self._store.counters.get(key, self._store.values.get(key))
|
||||
|
||||
|
||||
def _replica(redis: _SharedRedisFake) -> DualCache:
|
||||
return DualCache(redis_cache=redis, default_in_memory_ttl=600) # pyright: ignore[reportArgumentType] # duck-typed Redis double
|
||||
def _replica(redis: RedisCache) -> DualCache:
|
||||
return DualCache(redis_cache=redis, default_in_memory_ttl=600)
|
||||
|
||||
|
||||
class _Minter:
|
||||
|
|
@ -525,7 +490,7 @@ def test_token_unknown_team_grants_is_invalid_grant():
|
|||
|
||||
|
||||
def test_token_mints_on_a_replica_that_did_not_start_the_login():
|
||||
redis: Final = _SharedRedisFake()
|
||||
redis: Final = _SharedRedisCache()
|
||||
_set_cli_sso_flow(login_id=_SHARED_LOGIN_ID, cache=_replica(redis), flow=_completed_flow())
|
||||
|
||||
with _gateway_env(cache=_replica(redis)) as (client, _), patch(_MINT, return_value="sk-session") as mint:
|
||||
|
|
@ -537,7 +502,7 @@ def test_token_mints_on_a_replica_that_did_not_start_the_login():
|
|||
|
||||
|
||||
def test_token_refuses_a_device_code_another_replica_already_claimed():
|
||||
redis: Final = _SharedRedisFake()
|
||||
redis: Final = _SharedRedisCache()
|
||||
replica_a: Final = _replica(redis)
|
||||
_set_cli_sso_flow(login_id=_SHARED_LOGIN_ID, cache=replica_a, flow=_completed_flow())
|
||||
assert asyncio.run(gateway_endpoints._claim_device_code(_SHARED_LOGIN_ID, replica_a)) is True
|
||||
|
|
@ -616,7 +581,7 @@ def test_refresh_grant_mints_a_new_bearer_and_rotates_the_refresh_token():
|
|||
|
||||
|
||||
def test_refresh_grant_replay_is_refused_on_a_replica_that_did_not_serve_the_rotation():
|
||||
redis: Final = _SharedRedisFake()
|
||||
redis: Final = _SharedRedisCache()
|
||||
minter: Final = _Minter()
|
||||
with _gateway_env(claim_cache=_replica(redis), minter=minter) as (client, cache):
|
||||
refresh_token = str(_signed_in(client, cache)["refresh_token"])
|
||||
|
|
@ -761,7 +726,7 @@ def test_revoke_on_one_replica_ends_the_chain_on_another_through_a_namespaced_re
|
|||
with the configured ``redis_namespace``; the replica serving the next renewal must look for the
|
||||
marker under that same prefix, or the revoked chain keeps renewing everywhere but where it signed
|
||||
out."""
|
||||
redis: Final = _SharedRedisFake()
|
||||
redis: Final = _SharedRedisCache()
|
||||
minter: Final = _Minter()
|
||||
with _gateway_env(claim_cache=_replica(redis), minter=minter) as (client, cache):
|
||||
stored = str(_signed_in(client, cache)["refresh_token"])
|
||||
|
|
|
|||
|
|
@ -9,10 +9,13 @@ from http.cookies import SimpleCookie
|
|||
from typing import Final
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import fakeredis
|
||||
import pytest
|
||||
from redis.exceptions import ConnectionError as RedisConnectionError
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
import litellm
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
|
||||
from litellm.proxy.mcp_server.gateway_dcr_flow import (
|
||||
|
|
@ -63,6 +66,14 @@ from litellm.proxy.mcp_server.outbound_credentials.session_token import (
|
|||
mint_session_refresh_token,
|
||||
mint_session_token,
|
||||
)
|
||||
from litellm.redis.stores.mcp_single_use import (
|
||||
AssertionRedeemed,
|
||||
AuthCodeUsed,
|
||||
ConnectFlowUsed,
|
||||
RefreshFamilyRevoked,
|
||||
RefreshTokenUsed,
|
||||
)
|
||||
from tests._support.fake_redis import fake_redis_cache
|
||||
|
||||
MASTER_KEY = "sk-gateway-dcr-flow-tests"
|
||||
REDIRECT_URI = "https://claude.ai/api/mcp/auth_callback"
|
||||
|
|
@ -685,102 +696,130 @@ async def test_single_use_guard_in_memory_is_single_use_within_process():
|
|||
from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard
|
||||
|
||||
guard = _SingleUseGuard(DualCache()) # redis_cache is None
|
||||
assert await guard.claim("jti-inmem", 60) == "first"
|
||||
assert await guard.claim("jti-inmem", 60) == "replayed"
|
||||
assert await guard.claim(AuthCodeUsed("jti-inmem"), 60) == "first"
|
||||
assert await guard.claim(AuthCodeUsed("jti-inmem"), 60) == "replayed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_use_guard_uses_redis_as_sole_authority_when_configured():
|
||||
"""With Redis configured it is the SOLE authority: the shared INCR result decides the claim (1 →
|
||||
first caller, >1 → replay), and the per-worker in-memory count is never consulted."""
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
"""With Redis configured it is the SOLE authority: the shared count decides the claim (1 is the
|
||||
first caller, more is a replay), and the per-worker in-memory count is never written."""
|
||||
from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard
|
||||
|
||||
cache = DualCache()
|
||||
cache.redis_cache = MagicMock()
|
||||
cache.redis_cache.async_increment = AsyncMock(return_value=1)
|
||||
# in-memory must NOT be consulted when Redis is configured — poison it so any fallback is visible.
|
||||
cache.async_increment_cache = AsyncMock(side_effect=AssertionError("must not fall back to in-memory"))
|
||||
|
||||
server = fakeredis.FakeServer()
|
||||
cache = DualCache(redis_cache=fake_redis_cache(server))
|
||||
guard = _SingleUseGuard(cache)
|
||||
assert await guard.claim("jti-redis", 60) == "first"
|
||||
cache.redis_cache.async_increment = AsyncMock(return_value=2)
|
||||
assert await guard.claim("jti-redis", 60) == "replayed"
|
||||
|
||||
marker = AuthCodeUsed("jti-redis")
|
||||
assert await guard.claim(marker, 60) == "first"
|
||||
assert await guard.claim(marker, 60) == "replayed"
|
||||
assert await _SingleUseGuard(DualCache(redis_cache=fake_redis_cache(server))).claim(marker, 60) == "replayed"
|
||||
assert cache.in_memory_cache.get_cache("mcp_gateway_dcr_code_used:jti-redis") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_use_guard_writes_the_marker_the_old_increment_wrote_with_its_ttl():
|
||||
"""Old and new pods share the marker during a rolling deploy, so a claim writes the same
|
||||
INCRBYFLOAT string with the same TTL, and each side counts the other's claim."""
|
||||
from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard
|
||||
|
||||
server = fakeredis.FakeServer()
|
||||
redis_cache = fake_redis_cache(server, namespace="ns")
|
||||
raw = fakeredis.FakeAsyncRedis(server=server)
|
||||
guard = _SingleUseGuard(DualCache(redis_cache=redis_cache))
|
||||
|
||||
assert await guard.claim(RefreshTokenUsed("new-first"), 60) == "first"
|
||||
assert await raw.get("ns:mcp_gateway_dcr_refresh_used:new-first") == b"1"
|
||||
assert 59_000 < await raw.pttl("ns:mcp_gateway_dcr_refresh_used:new-first") <= 60_000
|
||||
assert await redis_cache.async_increment("mcp_gateway_dcr_refresh_used:new-first", 1, ttl=60) == 2.0
|
||||
|
||||
assert await redis_cache.async_increment("mcp_gateway_dcr_refresh_used:old-first", 1, ttl=60) == 1.0
|
||||
assert await guard.claim(RefreshTokenUsed("old-first"), 60) == "replayed"
|
||||
assert await guard.peek(RefreshTokenUsed("old-first")) == "claimed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_use_guard_keeps_the_ttl_a_marker_already_has():
|
||||
from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard
|
||||
|
||||
server = fakeredis.FakeServer()
|
||||
raw = fakeredis.FakeAsyncRedis(server=server)
|
||||
guard = _SingleUseGuard(DualCache(redis_cache=fake_redis_cache(server)))
|
||||
|
||||
assert await guard.claim(ConnectFlowUsed("jti-ttl"), 600) == "first"
|
||||
assert await guard.claim(ConnectFlowUsed("jti-ttl"), 5) == "replayed"
|
||||
assert 590_000 < await raw.pttl("mcp_gateway_dcr_flow_used:jti-ttl") <= 600_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_use_guard_fails_closed_when_redis_errors():
|
||||
"""A Redis fault must fail the claim CLOSED (refuse the id) rather than fall back to the per-worker
|
||||
in-memory count — which would let each replica observe count==1 and replay the one-time id (the
|
||||
Cursor/Veria replay-across-workers finding)."""
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
in-memory count, which would let each replica observe count==1 and replay the one-time id."""
|
||||
from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard
|
||||
|
||||
cache = DualCache()
|
||||
cache.redis_cache = MagicMock()
|
||||
cache.redis_cache.async_increment = AsyncMock(side_effect=ConnectionError("redis down"))
|
||||
cache.async_increment_cache = AsyncMock(return_value=1) # would fail OPEN if the guard fell back
|
||||
server = fakeredis.FakeServer()
|
||||
cache = DualCache(redis_cache=fake_redis_cache(server))
|
||||
server.connected = False
|
||||
|
||||
guard = _SingleUseGuard(cache)
|
||||
assert await guard.claim("jti-fault", 60) == "unavailable" # fail closed, not a fallback count of 1
|
||||
assert await _SingleUseGuard(cache).claim(AuthCodeUsed("jti-fault"), 60) == "unavailable"
|
||||
assert cache.in_memory_cache.get_cache("mcp_gateway_dcr_code_used:jti-fault") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_use_guard_peek_fails_closed_when_redis_errors():
|
||||
"""The peek that gates a refresh on its chain's revocation marker must read the Redis client
|
||||
itself: the cache wrapper's get swallows a fault into ``None``, which would make a revoked chain
|
||||
look unclaimed for exactly as long as Redis is down."""
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
"""The peek that gates a refresh on its chain's revocation marker must tell a Redis fault from a
|
||||
missing key, or a revoked chain looks unclaimed for exactly as long as Redis is down."""
|
||||
from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard
|
||||
|
||||
cache = DualCache()
|
||||
cache.redis_cache = MagicMock()
|
||||
cache.redis_cache.async_get_cache = AsyncMock(return_value=None)
|
||||
cache.redis_cache.check_and_fix_namespace = MagicMock(side_effect=lambda key: key)
|
||||
cache.redis_cache.init_async_client.return_value.get = AsyncMock(side_effect=ConnectionError("redis down"))
|
||||
server = fakeredis.FakeServer()
|
||||
guard = _SingleUseGuard(DualCache(redis_cache=fake_redis_cache(server)))
|
||||
assert await guard.claim(RefreshFamilyRevoked("family-fault"), 60) == "first"
|
||||
|
||||
guard = _SingleUseGuard(cache)
|
||||
assert await guard.peek("family-fault") == "unavailable"
|
||||
server.connected = False
|
||||
assert await guard.peek(RefreshFamilyRevoked("family-fault")) == "unavailable"
|
||||
|
||||
cache.redis_cache.init_async_client.return_value.get = AsyncMock(return_value=b"1")
|
||||
assert await guard.peek("family-fault") == "claimed"
|
||||
server.connected = True
|
||||
assert await guard.peek(RefreshFamilyRevoked("family-fault")) == "claimed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_use_guard_peek_reads_the_key_under_the_namespace_claim_wrote():
|
||||
"""A configured ``redis_namespace`` prefixes every key the cache wrapper writes, so the raw client
|
||||
read behind peek must ask for the same prefixed key: a revocation written under ``ns:...`` and
|
||||
peeked at the bare key would never be seen, and the revoked chain would keep renewing."""
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
"""A configured ``redis_namespace`` prefixes every key the cache writes, so peek must read the
|
||||
same prefixed key: a revocation written under ``ns:...`` and peeked at the bare key would never
|
||||
be seen, and the revoked chain would keep renewing."""
|
||||
from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard
|
||||
|
||||
stored: dict[str, int] = {} # mutable-ok: the fake Redis store the test inspects
|
||||
server = fakeredis.FakeServer()
|
||||
guard = _SingleUseGuard(DualCache(redis_cache=fake_redis_cache(server, namespace="ns")))
|
||||
|
||||
def namespaced(key: str) -> str:
|
||||
return f"ns:{key}"
|
||||
assert await guard.peek(RefreshFamilyRevoked("family-ns")) == "unclaimed"
|
||||
assert await guard.claim(RefreshFamilyRevoked("family-ns"), 60) == "first"
|
||||
assert await fakeredis.FakeAsyncRedis(server=server).keys("*") == [
|
||||
b"ns:mcp_gateway_dcr_refresh_family_revoked:family-ns"
|
||||
]
|
||||
assert await guard.peek(RefreshFamilyRevoked("family-ns")) == "claimed"
|
||||
|
||||
async def increment(key: str, value: int, ttl: int) -> int:
|
||||
stored[namespaced(key)] = stored.get(namespaced(key), 0) + value
|
||||
return stored[namespaced(key)]
|
||||
|
||||
async def raw_get(key: str) -> bytes | None:
|
||||
return None if key not in stored else str(stored[key]).encode()
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("marker", "key"),
|
||||
[
|
||||
(ConnectFlowUsed("j1"), b"mcp_gateway_dcr_flow_used:j1"),
|
||||
(AuthCodeUsed("j1"), b"mcp_gateway_dcr_code_used:j1"),
|
||||
(RefreshTokenUsed("j1"), b"mcp_gateway_dcr_refresh_used:j1"),
|
||||
(RefreshFamilyRevoked("f1"), b"mcp_gateway_dcr_refresh_family_revoked:f1"),
|
||||
(
|
||||
AssertionRedeemed(issuer="https://idp.example", jti="j1"),
|
||||
b"mcp_gateway_dcr:id_jag:" + hashlib.sha256(b"https://idp.example\nj1").hexdigest().encode(),
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_single_use_guard_claims_each_marker_under_the_key_old_pods_use(marker, key):
|
||||
from litellm.proxy.mcp_server.gateway_dcr_flow import _SingleUseGuard
|
||||
|
||||
cache = DualCache()
|
||||
cache.redis_cache = MagicMock()
|
||||
cache.redis_cache.check_and_fix_namespace = MagicMock(side_effect=namespaced)
|
||||
cache.redis_cache.async_increment = AsyncMock(side_effect=increment)
|
||||
cache.redis_cache.init_async_client.return_value.get = AsyncMock(side_effect=raw_get)
|
||||
|
||||
guard = _SingleUseGuard(cache)
|
||||
assert await guard.peek("family-ns") == "unclaimed"
|
||||
assert await guard.claim("family-ns", 60) == "first"
|
||||
assert set(stored) == {"ns:family-ns"}
|
||||
assert await guard.peek("family-ns") == "claimed"
|
||||
server = fakeredis.FakeServer()
|
||||
assert await _SingleUseGuard(DualCache(redis_cache=fake_redis_cache(server))).claim(marker, 60) == "first"
|
||||
assert await fakeredis.FakeAsyncRedis(server=server).keys("*") == [key]
|
||||
|
||||
|
||||
LOOPBACK_REDIRECT_URI = "http://localhost:3118/callback"
|
||||
|
|
@ -2247,38 +2286,25 @@ async def test_revoke_burns_the_refresh_token_and_answers_200_for_dead_or_unknow
|
|||
assert garbage.status_code == 200
|
||||
|
||||
|
||||
def _redis_that(async_increment, get=None):
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
cache = DualCache()
|
||||
cache.redis_cache = MagicMock()
|
||||
cache.redis_cache.async_increment = async_increment
|
||||
cache.redis_cache.check_and_fix_namespace = MagicMock(side_effect=lambda key: key)
|
||||
cache.redis_cache.init_async_client.return_value.get = get or AsyncMock(return_value=None)
|
||||
cache.redis_cache.async_get_cache = AsyncMock(
|
||||
side_effect=AssertionError("peek must read the client, not the wrapper")
|
||||
)
|
||||
cache.async_increment_cache = AsyncMock(side_effect=AssertionError("must not fall back to in-memory"))
|
||||
return cache
|
||||
def _shared_redis(server: fakeredis.FakeServer) -> DualCache:
|
||||
return DualCache(redis_cache=fake_redis_cache(server))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_revoke_answers_503_while_the_shared_record_cannot_be_written_then_burns_the_token():
|
||||
"""A revocation whose single-use marker never reached Redis must not report success: the token is
|
||||
still redeemable on every worker, so the client has to hear 503 (RFC 7009 2.2.1) and retry."""
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
client_id = (await _register([LOOPBACK_REDIRECT_URI]))["client_id"]
|
||||
issued = DualCache()
|
||||
payload = json.loads(
|
||||
(await _redeem_native(await _native_code(client_id, cache=issued), client_id, _Minter(), cache=issued)).body
|
||||
)
|
||||
server = fakeredis.FakeServer()
|
||||
shared = _shared_redis(server)
|
||||
|
||||
server.connected = False
|
||||
redis_down = await revoke_refresh_token(
|
||||
token=payload["refresh_token"],
|
||||
client_id=client_id,
|
||||
master_key=MASTER_KEY,
|
||||
cache=_redis_that(AsyncMock(side_effect=ConnectionError("redis down"))),
|
||||
token=payload["refresh_token"], client_id=client_id, master_key=MASTER_KEY, cache=shared
|
||||
)
|
||||
assert redis_down.status_code == 503
|
||||
assert json.loads(redis_down.body) == {
|
||||
|
|
@ -2287,47 +2313,32 @@ async def test_revoke_answers_503_while_the_shared_record_cannot_be_written_then
|
|||
}
|
||||
assert redis_down.headers["cache-control"] == "no-store"
|
||||
|
||||
server.connected = True
|
||||
redis_back = await revoke_refresh_token(
|
||||
token=payload["refresh_token"],
|
||||
client_id=client_id,
|
||||
master_key=MASTER_KEY,
|
||||
cache=_redis_that(AsyncMock(return_value=1)),
|
||||
token=payload["refresh_token"], client_id=client_id, master_key=MASTER_KEY, cache=shared
|
||||
)
|
||||
assert redis_back.status_code == 200
|
||||
assert json.loads(redis_back.body) == {}
|
||||
|
||||
already_burned = await revoke_refresh_token(
|
||||
token=payload["refresh_token"],
|
||||
client_id=client_id,
|
||||
master_key=MASTER_KEY,
|
||||
cache=_redis_that(AsyncMock(return_value=2)),
|
||||
token=payload["refresh_token"], client_id=client_id, master_key=MASTER_KEY, cache=shared
|
||||
)
|
||||
assert already_burned.status_code == 200
|
||||
|
||||
|
||||
class _RedisThatFaultsOnTheSecondWrite:
|
||||
"""A shared Redis double whose first increment lands and whose second raises, the fault between
|
||||
revoke's two writes; its raw client reads back exactly the keys that landed, and later writes land."""
|
||||
class _RedisThatFaultsOnTheSecondClaim(fakeredis.FakeAsyncRedis):
|
||||
"""A shared Redis whose second single-use claim faults, the fault between revoke's two writes;
|
||||
every other command, and every later claim, lands."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.landed: tuple[str, ...] = ()
|
||||
self.writes = 0
|
||||
def __init__(self, server: fakeredis.FakeServer) -> None:
|
||||
super().__init__(server=server)
|
||||
self.claims = 0
|
||||
|
||||
def check_and_fix_namespace(self, key: str) -> str:
|
||||
return key
|
||||
|
||||
def init_async_client(self) -> "_RedisThatFaultsOnTheSecondWrite":
|
||||
return self
|
||||
|
||||
async def async_increment(self, key: str, value: int, **kwargs: object) -> int:
|
||||
self.writes += 1
|
||||
if self.writes == 2:
|
||||
raise ConnectionError("redis down")
|
||||
self.landed = (*self.landed, key)
|
||||
return self.landed.count(key)
|
||||
|
||||
async def get(self, key: str) -> int | None:
|
||||
return self.landed.count(key) or None
|
||||
async def evalsha(self, sha: str, numkeys: int, *keys_and_args: str | bytes | float) -> object:
|
||||
self.claims += 1
|
||||
if self.claims == 2:
|
||||
raise RedisConnectionError("redis down")
|
||||
return await super().evalsha(sha, numkeys, *keys_and_args)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -2343,8 +2354,13 @@ async def test_revoke_ends_the_chain_first_so_a_fault_before_the_jti_burn_leaves
|
|||
copied_and_rotated = json.loads(
|
||||
(await _refresh_native(payload["refresh_token"], client_id, _Minter(), issued)).body
|
||||
)
|
||||
redis: Final = _RedisThatFaultsOnTheSecondWrite()
|
||||
shared: Final = DualCache(redis_cache=redis) # pyright: ignore[reportArgumentType] # duck-typed Redis double
|
||||
server: Final = fakeredis.FakeServer()
|
||||
redis_cache: Final = fake_redis_cache(server)
|
||||
litellm.in_memory_llm_clients_cache.set_cache(
|
||||
key=redis_cache._get_async_client_cache_key(), # pyright: ignore[reportPrivateUsage] # seeds the per-loop client slot RedisCache reads
|
||||
value=_RedisThatFaultsOnTheSecondClaim(server),
|
||||
)
|
||||
shared: Final = DualCache(redis_cache=redis_cache)
|
||||
|
||||
half_written = await revoke_refresh_token(
|
||||
token=payload["refresh_token"], client_id=client_id, master_key=MASTER_KEY, cache=shared
|
||||
|
|
@ -2368,33 +2384,26 @@ async def test_refresh_answers_503_without_burning_the_token_while_redis_is_down
|
|||
"""Fail closed, but say why: a refresh the shared backend could not record is refused with 503
|
||||
rather than ``invalid_grant``, so the CLI keeps the key it has and retries instead of telling the
|
||||
user the token was already used and sending them back through the browser."""
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
client_id = (await _register([LOOPBACK_REDIRECT_URI]))["client_id"]
|
||||
issued = DualCache()
|
||||
payload = json.loads(
|
||||
(await _redeem_native(await _native_code(client_id, cache=issued), client_id, _Minter(), cache=issued)).body
|
||||
)
|
||||
server = fakeredis.FakeServer()
|
||||
shared = _shared_redis(server)
|
||||
|
||||
redis_down = await _refresh_native(
|
||||
payload["refresh_token"],
|
||||
client_id,
|
||||
_Minter(),
|
||||
_redis_that(AsyncMock(side_effect=ConnectionError("redis down"))),
|
||||
)
|
||||
server.connected = False
|
||||
redis_down = await _refresh_native(payload["refresh_token"], client_id, _Minter(), shared)
|
||||
assert redis_down.status_code == 503
|
||||
assert json.loads(redis_down.body)["error"] == "temporarily_unavailable"
|
||||
assert "refresh_token" not in json.loads(redis_down.body)
|
||||
|
||||
redis_back = await _refresh_native(
|
||||
payload["refresh_token"], client_id, _Minter(), _redis_that(AsyncMock(return_value=1))
|
||||
)
|
||||
server.connected = True
|
||||
redis_back = await _refresh_native(payload["refresh_token"], client_id, _Minter(), shared)
|
||||
assert redis_back.status_code == 200
|
||||
assert json.loads(redis_back.body)["refresh_token"] != payload["refresh_token"]
|
||||
|
||||
replayed = await _refresh_native(
|
||||
payload["refresh_token"], client_id, _Minter(), _redis_that(AsyncMock(return_value=2))
|
||||
)
|
||||
replayed = await _refresh_native(payload["refresh_token"], client_id, _Minter(), shared)
|
||||
assert replayed.status_code == 400
|
||||
assert json.loads(replayed.body)["error"] == "invalid_grant"
|
||||
|
||||
|
|
@ -2403,32 +2412,26 @@ async def test_refresh_answers_503_without_burning_the_token_while_redis_is_down
|
|||
async def test_refresh_of_a_rotated_token_answers_503_before_minting_while_redis_cannot_be_read():
|
||||
"""A rotated token's chain marker is read before anything is minted or claimed; a Redis read fault
|
||||
is a 503, never a pass: the token stays unburned and unminted until Redis answers."""
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
client_id = (await _register([LOOPBACK_REDIRECT_URI]))["client_id"]
|
||||
issued = DualCache()
|
||||
payload = json.loads(
|
||||
(await _redeem_native(await _native_code(client_id, cache=issued), client_id, _Minter(), cache=issued)).body
|
||||
)
|
||||
rotated = json.loads(
|
||||
(
|
||||
await _refresh_native(
|
||||
payload["refresh_token"], client_id, _Minter(), _redis_that(AsyncMock(return_value=1))
|
||||
)
|
||||
).body
|
||||
)["refresh_token"]
|
||||
server = fakeredis.FakeServer()
|
||||
shared = _shared_redis(server)
|
||||
rotated = json.loads((await _refresh_native(payload["refresh_token"], client_id, _Minter(), shared)).body)[
|
||||
"refresh_token"
|
||||
]
|
||||
|
||||
minter = _Minter()
|
||||
claim = AsyncMock(return_value=1)
|
||||
unreadable = await _refresh_native(
|
||||
rotated, client_id, minter, _redis_that(claim, get=AsyncMock(side_effect=ConnectionError("redis down")))
|
||||
)
|
||||
server.connected = False
|
||||
unreadable = await _refresh_native(rotated, client_id, minter, shared)
|
||||
assert unreadable.status_code == 503
|
||||
assert json.loads(unreadable.body)["error"] == "temporarily_unavailable"
|
||||
assert minter.calls == []
|
||||
assert claim.await_count == 0
|
||||
|
||||
readable = await _refresh_native(rotated, client_id, _Minter(), _redis_that(AsyncMock(return_value=1)))
|
||||
server.connected = True
|
||||
readable = await _refresh_native(rotated, client_id, _Minter(), shared)
|
||||
assert readable.status_code == 200
|
||||
assert json.loads(readable.body)["refresh_token"] != rotated
|
||||
|
||||
|
|
@ -2724,15 +2727,17 @@ async def test_introspect_fails_closed_on_dead_user_and_503s_on_outage():
|
|||
@pytest.mark.asyncio
|
||||
async def test_introspect_503s_while_the_single_use_record_cannot_be_read():
|
||||
"""A token whose chain may have been revoked is never reported active on a Redis read fault."""
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
keys, now, principal = _introspection_fixtures()
|
||||
minted = mint_session_refresh_token(principal, keys, now)
|
||||
unreadable = _redis_that(AsyncMock(return_value=1), get=AsyncMock(side_effect=ConnectionError("redis down")))
|
||||
status, body = await _introspect(minted.token.get_secret_value(), cache=unreadable)
|
||||
server = fakeredis.FakeServer()
|
||||
shared = _shared_redis(server)
|
||||
|
||||
server.connected = False
|
||||
status, body = await _introspect(minted.token.get_secret_value(), cache=shared)
|
||||
assert (status, body["error"]) == (503, "temporarily_unavailable")
|
||||
|
||||
status, body = await _introspect(minted.token.get_secret_value(), cache=_redis_that(AsyncMock(return_value=1)))
|
||||
server.connected = True
|
||||
status, body = await _introspect(minted.token.get_secret_value(), cache=shared)
|
||||
assert (status, body["active"]) == (200, True)
|
||||
|
||||
|
||||
|
|
@ -3308,7 +3313,7 @@ REQUESTING_APP = "0oa-requesting-app"
|
|||
ASSERTED = AssertionIdentity(
|
||||
user_id="u1",
|
||||
team_id="team-b",
|
||||
replay_key="mcp_gateway_dcr:id_jag:jti-1",
|
||||
replay=AssertionRedeemed(issuer="https://idp.example", jti="jti-1"),
|
||||
replay_ttl_seconds=300,
|
||||
scope="litellm:mcp",
|
||||
allowed_routes=("mcp_inference_routes",),
|
||||
|
|
@ -3359,7 +3364,11 @@ async def test_jwt_bearer_grant_mints_the_proxy_credential_for_the_asserted_user
|
|||
assert replayed.status_code == 400
|
||||
assert json.loads(replayed.body)["error"] == "invalid_grant"
|
||||
assert minter.calls == [("u1", "team-b")]
|
||||
fresh = _Redeemer(ASSERTED.model_copy(update={"replay_key": "mcp_gateway_dcr:id_jag:jti-2", "scope": None}))
|
||||
fresh = _Redeemer(
|
||||
ASSERTED.model_copy(
|
||||
update={"replay": AssertionRedeemed(issuer="https://idp.example", jti="jti-2"), "scope": None}
|
||||
)
|
||||
)
|
||||
again = await _redeem_assertion_native(minter, fresh, cache=cache)
|
||||
assert again.status_code == 200
|
||||
assert "scope" not in json.loads(again.body)
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from litellm.proxy.mcp_server.id_jag_grant import (
|
|||
select_team,
|
||||
subject_resolver,
|
||||
)
|
||||
from litellm.redis.stores.mcp_single_use import marker_key
|
||||
|
||||
ISSUER = "https://idp.example.com"
|
||||
GATEWAY = "https://gateway.example.com"
|
||||
|
|
@ -204,11 +205,14 @@ async def test_a_valid_id_jag_names_the_user_its_single_use_key_and_its_scope():
|
|||
assert resolver.calls == ((_claims(), "team-b"),)
|
||||
assert (identity.user_id, identity.team_id, identity.scope) == ("u1", "team-b", "litellm:mcp")
|
||||
assert identity.allowed_routes == ("mcp_inference_routes",)
|
||||
assert identity.replay_key == "mcp_gateway_dcr:id_jag:" + hashlib.sha256(f"{ISSUER}\njti-1".encode()).hexdigest()
|
||||
assert (
|
||||
marker_key(identity.replay)
|
||||
== "mcp_gateway_dcr:id_jag:" + hashlib.sha256(f"{ISSUER}\njti-1".encode()).hexdigest()
|
||||
)
|
||||
assert identity.replay_ttl_seconds == math.ceil(300)
|
||||
other_jti = await _identity(_redemption(_assertion(jti="jti-2")), _Verifier(), _Resolver())
|
||||
assert isinstance(other_jti, AssertionIdentity)
|
||||
assert other_jti.replay_key != identity.replay_key
|
||||
assert marker_key(other_jti.replay) != marker_key(identity.replay)
|
||||
teamless = await _identity(_redemption(_assertion(scope=""), team_id=None))
|
||||
assert isinstance(teamless, AssertionIdentity)
|
||||
assert (teamless.team_id, teamless.scope, teamless.allowed_routes) == (None, None, None)
|
||||
|
|
|
|||
|
|
@ -201,7 +201,7 @@ def test_a_targeted_coroutine_closed_from_another_context_does_not_raise() -> No
|
|||
assert current_service_target() is None
|
||||
|
||||
|
||||
_DIRECT_REDIS_CALL: Final = re.compile(r"\b_?redis_cache\.(?!async_register_script\b)(?:async_)?\w+\(")
|
||||
_DIRECT_REDIS_CALL: Final = re.compile(r"\b_?redis_cache\.(?!(?:async_register_script|transport)\b)(?:async_)?\w+\(")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -226,6 +226,22 @@ def _scan_function(source: str, fn: ast.FunctionDef | ast.AsyncFunctionDef) -> _
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "reaches_redis"),
|
||||
(
|
||||
("redis_cache.transport()", False),
|
||||
('redis_cache.async_register_script("lua")', False),
|
||||
('redis_cache.async_get_cache("key")', True),
|
||||
('redis_cache.transport(); redis_cache.async_get_cache("key")', True),
|
||||
),
|
||||
)
|
||||
def test_redis_family_scan_distinguishes_factories_from_commands(body: str, reaches_redis: bool) -> None:
|
||||
source: Final = f"def call():\n {body}\n"
|
||||
function: Final = ast.parse(source).body[0]
|
||||
assert isinstance(function, ast.FunctionDef)
|
||||
assert _scan_function(source, function).reaches_redis_directly is reaches_redis
|
||||
|
||||
|
||||
def _covered_by_callers(scans: tuple[_FunctionScan, ...], covered: frozenset[str]) -> frozenset[str]:
|
||||
"""Close ``covered`` over functions whose every in-file caller already declares a family."""
|
||||
callers: Final = {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue