mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 424eb0415e into e16d79d635
This commit is contained in:
commit
fb57e89701
5 changed files with 360 additions and 30 deletions
|
|
@ -1900,6 +1900,26 @@ async def get_user_oauth_credential(
|
|||
return _parse_oauth_payload(decoded)
|
||||
|
||||
|
||||
async def delete_user_oauth_credential_holding_refresh_token(
|
||||
prisma_client: PrismaClient,
|
||||
user_id: str,
|
||||
server_id: str,
|
||||
refresh_token: str,
|
||||
) -> bool:
|
||||
"""Delete the user's OAuth credential only while it still holds ``refresh_token``. The delete is
|
||||
keyed on the exact ciphertext read, so a re-authorization that lands concurrently survives."""
|
||||
row: Final = await _db_find_user_credential_row(prisma_client, user_id, server_id)
|
||||
if row is None:
|
||||
return False
|
||||
payload: Final = decode_oauth_payload(row.credential_b64)
|
||||
if payload is None or payload.get("refresh_token") != refresh_token:
|
||||
return False
|
||||
deleted: Final = await _user_credential_actions(prisma_client).delete_many(
|
||||
where={"user_id": user_id, "server_id": server_id, "credential_b64": row.credential_b64}
|
||||
)
|
||||
return deleted > 0
|
||||
|
||||
|
||||
def _server_user_credential_item(
|
||||
row: "prisma_db_models.LiteLLM_MCPUserCredentials",
|
||||
) -> MCPServerUserCredentialListItem:
|
||||
|
|
|
|||
|
|
@ -11,10 +11,11 @@ across replicas - is the wrapping store's distributed single-flight, not this re
|
|||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from typing import TYPE_CHECKING, Final, Protocol, TypeAlias
|
||||
|
||||
from fastapi import HTTPException
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import (
|
||||
|
|
@ -29,12 +30,29 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oa
|
|||
from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
|
||||
OAuthToken,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Result
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import (
|
||||
RefreshFailure,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
ServerLookup = Callable[[str], "MCPServer | None"]
|
||||
TokenEndpointPost = Callable[[str, dict[str, str], dict[str, str]], Awaitable["dict[str, object] | None"]]
|
||||
TokenEndpointPost: TypeAlias = Callable[
|
||||
[str, dict[str, str], dict[str, str]], Awaitable[Result[Mapping[str, object], RefreshFailure]]
|
||||
]
|
||||
CredentialPurge: TypeAlias = Callable[[str, str, str], Awaitable[None]]
|
||||
|
||||
_REVOKED_GRANT_ERRORS: Final = frozenset({"invalid_grant", "invalid_token"})
|
||||
|
||||
|
||||
def classify_token_endpoint_refusal(status_code: int, error: str | None) -> RefreshFailure:
|
||||
if 400 <= status_code < 500 and error in _REVOKED_GRANT_ERRORS:
|
||||
return RefreshFailure.of_rejected(f"the token endpoint answered {status_code} {error}")
|
||||
return RefreshFailure.of_unavailable(
|
||||
f"the token endpoint answered {status_code} {error or 'without an error code'}"
|
||||
)
|
||||
|
||||
|
||||
class CredentialPersist(Protocol):
|
||||
|
|
@ -71,13 +89,15 @@ def _parse_scopes(raw: object) -> tuple[str, ...] | None:
|
|||
class AuthorizationCodeRefresher:
|
||||
"""``TokenRefresher`` for authorization_code: refresh_token grant against the server, then persist.
|
||||
|
||||
``token_endpoint`` POSTs the OAuth form and returns the parsed JSON body (``None`` on any
|
||||
transport/HTTP failure, mirroring v1: a failed refresh is a miss, not a 500). ``persist`` writes
|
||||
the rotated triple for ``(user, server)`` - the v1 ``store_user_oauth_credential`` write, which
|
||||
stays. Returns ``None`` (the arm challenges) when there is no refresh_token, the server lacks a
|
||||
token endpoint, or the grant fails; never a stale or partial token. A rotated refresh_token from
|
||||
the response replaces the old one; an omitted one is carried forward, as are the recorded scopes
|
||||
when the response omits ``scope``.
|
||||
``token_endpoint`` POSTs the OAuth form and returns the parsed JSON body or a classified
|
||||
``RefreshFailure``. ``persist`` writes the rotated triple for ``(user, server)`` - the v1
|
||||
``store_user_oauth_credential`` write, which stays. A ``rejected`` failure (the IdP answered
|
||||
``invalid_grant`` / ``invalid_token``) means the stored refresh token is dead, so ``purge`` drops
|
||||
the credential that still holds it and the user is challenged to re-authorize; an ``unavailable``
|
||||
one keeps the credential so the next request retries. Returns ``None`` (the arm challenges) when
|
||||
there is no refresh_token, the server lacks a token endpoint, or the grant fails; never a stale or
|
||||
partial token. A rotated refresh_token from the response replaces the old one; an omitted one is
|
||||
carried forward, as are the recorded scopes when the response omits ``scope``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
|
@ -86,12 +106,14 @@ class AuthorizationCodeRefresher:
|
|||
token_endpoint: TokenEndpointPost,
|
||||
persist: CredentialPersist,
|
||||
*,
|
||||
purge: CredentialPurge,
|
||||
clock: Callable[[], float] = time.time,
|
||||
identity_validator: BindingValidator = enforce_oauth_identity_binding,
|
||||
) -> None:
|
||||
self._server_lookup = server_lookup
|
||||
self._token_endpoint = token_endpoint
|
||||
self._persist = persist
|
||||
self._purge = purge
|
||||
self._clock = clock
|
||||
self._identity_validator = identity_validator
|
||||
|
||||
|
|
@ -139,9 +161,11 @@ class AuthorizationCodeRefresher:
|
|||
"refresh_token": token.refresh_token,
|
||||
**token_request.body,
|
||||
}
|
||||
body: Final = await self._token_endpoint(token_url, form, token_request.headers)
|
||||
if body is None:
|
||||
outcome: Final = await self._token_endpoint(token_url, form, token_request.headers)
|
||||
if isinstance(outcome, Error):
|
||||
await self._on_failure(user_id, server_id, token.refresh_token, outcome.error)
|
||||
return None
|
||||
body: Final = outcome.ok
|
||||
access_token: Final = body.get("access_token")
|
||||
if not isinstance(access_token, str) or not access_token:
|
||||
return None
|
||||
|
|
@ -177,3 +201,24 @@ class AuthorizationCodeRefresher:
|
|||
identity_binding_proof=binding_proof,
|
||||
cimd_client_id=cimd_client_id,
|
||||
)
|
||||
|
||||
async def _on_failure(self, user_id: str, server_id: str, refresh_token: str, failure: RefreshFailure) -> None:
|
||||
match failure.kind:
|
||||
case "rejected":
|
||||
verbose_logger.warning(
|
||||
"MCP OAuth refresh rejected for user=%s server=%s (%s); removing the stored credential so "
|
||||
"the user re-authorizes",
|
||||
user_id,
|
||||
server_id,
|
||||
failure.detail,
|
||||
)
|
||||
await self._purge(user_id, server_id, refresh_token)
|
||||
case "unavailable":
|
||||
verbose_logger.warning(
|
||||
"MCP OAuth refresh failed for user=%s server=%s (%s); keeping the stored credential",
|
||||
user_id,
|
||||
server_id,
|
||||
failure.detail,
|
||||
)
|
||||
case _:
|
||||
assert_never(failure.kind)
|
||||
|
|
|
|||
|
|
@ -15,10 +15,14 @@ from collections.abc import Callable, Mapping
|
|||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.proxy._experimental.mcp_server.oauth_identity_binding import credential_binding_matches
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import (
|
||||
AuthorizationCodeRefresher,
|
||||
classify_token_endpoint_refusal,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.dual_cache_token_backend import (
|
||||
AsyncCache,
|
||||
|
|
@ -33,9 +37,13 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_sto
|
|||
TokenCacheBackend,
|
||||
TokenStoreUnavailable,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.runtime_refresh_coordinator import (
|
||||
runtime_refresh_coordinator,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import (
|
||||
RefreshFailure,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import (
|
||||
OAuthTokenCacheCodec,
|
||||
)
|
||||
|
|
@ -95,7 +103,25 @@ async def _persist_credential(
|
|||
)
|
||||
|
||||
|
||||
async def _post_token_endpoint(url: str, form: dict[str, str], headers: dict[str, str]) -> dict[str, object] | None:
|
||||
async def _purge_credential(user_id: str, server_id: str, refresh_token: str) -> None:
|
||||
from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # avoids a proxy_server import cycle
|
||||
delete_user_oauth_credential_holding_refresh_token,
|
||||
)
|
||||
from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # avoids a proxy_server import cycle
|
||||
|
||||
if prisma_client is None:
|
||||
return
|
||||
try:
|
||||
await delete_user_oauth_credential_holding_refresh_token(prisma_client, user_id, server_id, refresh_token)
|
||||
except Exception as exc: # noqa: BLE001 # the refresh already failed closed; the next refusal retries the purge
|
||||
verbose_logger.warning(
|
||||
"Failed to remove revoked MCP OAuth credential for user=%s server=%s: %s", user_id, server_id, exc
|
||||
)
|
||||
|
||||
|
||||
async def _post_token_endpoint(
|
||||
url: str, form: dict[str, str], headers: dict[str, str]
|
||||
) -> Result[Mapping[str, object], RefreshFailure]:
|
||||
from litellm.llms.custom_httpx.http_handler import ( # noqa: PLC0415
|
||||
get_async_httpx_client, # pyright: ignore
|
||||
)
|
||||
|
|
@ -105,17 +131,35 @@ async def _post_token_endpoint(url: str, form: dict[str, str], headers: dict[str
|
|||
# object and the refresher validates each field, so the untyped boundary is contained here.
|
||||
provider: Final = httpxSpecialProvider.Oauth2Check
|
||||
request_headers: Final = {"Accept": "application/json", **headers}
|
||||
# A failed refresh is a miss, not a 500 (matches v1), so any error becomes None.
|
||||
try:
|
||||
client: Final = get_async_httpx_client(llm_provider=provider) # pyright: ignore
|
||||
response: Final = await client.post(url, headers=request_headers, data=form) # pyright: ignore
|
||||
response.raise_for_status() # pyright: ignore
|
||||
body: Final[dict[str, object]] = response.json() # pyright: ignore
|
||||
content: Final[bytes] = response.content # pyright: ignore
|
||||
except httpx.HTTPStatusError as exc:
|
||||
return Error(classify_token_endpoint_refusal(exc.response.status_code, _token_error_code(exc.response.content)))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
verbose_logger.warning("MCP OAuth refresh request failed: %s", exc)
|
||||
return Error(RefreshFailure.of_unavailable(f"the token endpoint request failed ({type(exc).__name__})"))
|
||||
try:
|
||||
return Ok(_TOKEN_RESPONSE.validate_json(content))
|
||||
except ValidationError:
|
||||
return Error(RefreshFailure.of_unavailable("the token endpoint returned a malformed response"))
|
||||
|
||||
|
||||
class _TokenErrorBody(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
error: str | None = None
|
||||
|
||||
|
||||
_TOKEN_RESPONSE: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _token_error_code(content: bytes) -> str | None:
|
||||
try:
|
||||
return _TokenErrorBody.model_validate_json(content).error
|
||||
except ValidationError:
|
||||
return None
|
||||
else:
|
||||
return body # pyright: ignore
|
||||
|
||||
|
||||
def _redis_cache_is_available() -> bool:
|
||||
|
|
@ -164,7 +208,9 @@ def _build_per_user_oauth_token_store(
|
|||
server_lookup: ServerLookup,
|
||||
) -> tuple[CachedOAuthTokenStore, bool]:
|
||||
backend, coordinator, uses_redis = _runtime_backend_and_coordinator()
|
||||
refresher: Final = AuthorizationCodeRefresher(server_lookup, _post_token_endpoint, _persist_credential)
|
||||
refresher: Final = AuthorizationCodeRefresher(
|
||||
server_lookup, _post_token_endpoint, _persist_credential, purge=_purge_credential
|
||||
)
|
||||
refreshing: Final = RefreshingTokenStore(
|
||||
V2PerUserTokenStore(partial(_read_bound_credential, server_lookup)), refresher, coordinator=coordinator
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,10 +4,15 @@ import pytest
|
|||
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import (
|
||||
AuthorizationCodeRefresher,
|
||||
classify_token_endpoint_refusal,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
|
||||
OAuthToken,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import (
|
||||
RefreshFailure,
|
||||
)
|
||||
|
||||
|
||||
class _Server:
|
||||
|
|
@ -45,11 +50,27 @@ def _endpoint(body, sink=None):
|
|||
async def post(url, form, headers):
|
||||
if sink is not None:
|
||||
sink.append((url, form, headers))
|
||||
return body
|
||||
if body is None:
|
||||
return Error(RefreshFailure.of_unavailable("token endpoint down"))
|
||||
return Ok(body)
|
||||
|
||||
return post
|
||||
|
||||
|
||||
def _failing_endpoint(failure):
|
||||
async def post(url, form, headers):
|
||||
return Error(failure)
|
||||
|
||||
return post
|
||||
|
||||
|
||||
def _recording_purge(sink):
|
||||
async def purge(user_id, server_id, refresh_token):
|
||||
sink.append((user_id, server_id, refresh_token))
|
||||
|
||||
return purge
|
||||
|
||||
|
||||
def _recording_persist(sink):
|
||||
async def persist(
|
||||
user_id, server_id, access_token, refresh_token, expires_in, scopes, cimd_client_id=None
|
||||
|
|
@ -62,12 +83,13 @@ def _recording_persist(sink):
|
|||
|
||||
|
||||
def _refresher(
|
||||
server=None, body=None, *, post_sink=None, persist_sink=None, clock=lambda: 1000.0
|
||||
server=None, body=None, *, post_sink=None, persist_sink=None, purge_sink=None, clock=lambda: 1000.0
|
||||
):
|
||||
return AuthorizationCodeRefresher(
|
||||
_lookup(server if server is not None else _Server()),
|
||||
_endpoint(body, post_sink),
|
||||
_recording_persist(persist_sink if persist_sink is not None else []),
|
||||
purge=_recording_purge(purge_sink if purge_sink is not None else []),
|
||||
clock=clock,
|
||||
)
|
||||
|
||||
|
|
@ -184,6 +206,55 @@ async def test_grant_failure_does_not_persist():
|
|||
assert persisted == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error_code", ["invalid_grant", "invalid_token"])
|
||||
@pytest.mark.parametrize("status_code", [400, 401])
|
||||
async def test_revoked_refresh_token_purges_the_credential_that_holds_it(status_code, error_code):
|
||||
persisted = []
|
||||
purged = []
|
||||
refresher = AuthorizationCodeRefresher(
|
||||
_lookup(_Server()),
|
||||
_failing_endpoint(classify_token_endpoint_refusal(status_code, error_code)),
|
||||
_recording_persist(persisted),
|
||||
purge=_recording_purge(purged),
|
||||
)
|
||||
assert await refresher.refresh("alice", "srv", OAuthToken("old", refresh_token="revoked-rt")) is None
|
||||
assert purged == [("alice", "srv", "revoked-rt")]
|
||||
assert persisted == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"status_code, error_code",
|
||||
[
|
||||
(503, "temporarily_unavailable"),
|
||||
(500, "invalid_grant"),
|
||||
(429, "slow_down"),
|
||||
(400, "invalid_client"),
|
||||
(400, "invalid_request"),
|
||||
(400, None),
|
||||
],
|
||||
)
|
||||
async def test_transient_or_non_grant_refusal_keeps_the_credential(status_code, error_code):
|
||||
purged = []
|
||||
refresher = AuthorizationCodeRefresher(
|
||||
_lookup(_Server()),
|
||||
_failing_endpoint(classify_token_endpoint_refusal(status_code, error_code)),
|
||||
_recording_persist([]),
|
||||
purge=_recording_purge(purged),
|
||||
)
|
||||
assert await refresher.refresh("alice", "srv", OAuthToken("old", refresh_token="rt")) is None
|
||||
assert purged == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unreachable_token_endpoint_keeps_the_credential():
|
||||
purged = []
|
||||
refresher = _refresher(body=None, purge_sink=purged)
|
||||
assert await refresher.refresh("alice", "srv", OAuthToken("old", refresh_token="rt")) is None
|
||||
assert purged == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_without_access_token_does_not_persist():
|
||||
persisted = []
|
||||
|
|
@ -304,6 +375,7 @@ async def test_identity_rejection_never_persists_or_returns_refreshed_token():
|
|||
_lookup(_Server()),
|
||||
_endpoint({"access_token": "foreign-token", "id_token": "foreign-identity"}),
|
||||
persist,
|
||||
purge=_recording_purge([]),
|
||||
identity_validator=validator,
|
||||
)
|
||||
assert await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) is None
|
||||
|
|
@ -320,6 +392,7 @@ async def test_verified_refresh_preserves_binding_proof_in_storage():
|
|||
_lookup(_Server()),
|
||||
_endpoint({"access_token": "new", "refresh_token": "rotated"}),
|
||||
persist,
|
||||
purge=_recording_purge([]),
|
||||
identity_validator=validator,
|
||||
)
|
||||
token = await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt"))
|
||||
|
|
@ -394,7 +467,7 @@ async def test_saved_cimd_identity_refreshes_without_discovery(monkeypatch):
|
|||
assert form["client_id"] == identity
|
||||
assert "client_secret" not in form
|
||||
assert "Authorization" not in headers
|
||||
return {"access_token": "new", "refresh_token": "rotated"}
|
||||
return Ok({"access_token": "new", "refresh_token": "rotated"})
|
||||
|
||||
server = MCPServer(
|
||||
server_id="srv", name="srv", transport="http", auth_type="oauth2", oauth2_flow="authorization_code",
|
||||
|
|
@ -402,7 +475,9 @@ async def test_saved_cimd_identity_refreshes_without_discovery(monkeypatch):
|
|||
)
|
||||
token = await V2PerUserTokenStore(read_credential).fetch("alice", "srv")
|
||||
assert token is not None
|
||||
refreshed = await AuthorizationCodeRefresher(lambda _: server, post, persist).refresh("alice", "srv", token)
|
||||
refreshed = await AuthorizationCodeRefresher(
|
||||
lambda _: server, post, persist, purge=_recording_purge([])
|
||||
).refresh("alice", "srv", token)
|
||||
assert refreshed is not None
|
||||
assert refreshed.access_token == "new"
|
||||
assert refreshed.refresh_token == "rotated"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import (
|
||||
|
|
@ -35,6 +36,8 @@ def credential_database(monkeypatch):
|
|||
prisma = MagicMock()
|
||||
table = prisma.db.litellm_mcpusercredentials
|
||||
table.upsert = AsyncMock()
|
||||
table.find_unique = AsyncMock(return_value=None)
|
||||
table.delete_many = AsyncMock(return_value=1)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "credential-persistence-test-salt")
|
||||
return table
|
||||
|
|
@ -59,23 +62,39 @@ async def test_credential_persistence_preserves_known_and_unknown_scopes(credent
|
|||
[((), None, []), (("read", "write"), "", []), (("read", "write"), None, ["read", "write"])],
|
||||
)
|
||||
async def test_native_refresh_persists_actual_grant_through_storage_adapter(
|
||||
credential_database, prior_scopes, response_scope, expected,
|
||||
credential_database,
|
||||
prior_scopes,
|
||||
response_scope,
|
||||
expected,
|
||||
):
|
||||
from litellm.proxy._experimental.mcp_server.db import decode_oauth_payload
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import AuthorizationCodeRefresher
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import _persist_credential
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import (
|
||||
AuthorizationCodeRefresher,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import (
|
||||
_persist_credential,
|
||||
_purge_credential,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
server = MCPServer(
|
||||
server_id="srv", name="srv", transport="http", auth_type="oauth2",
|
||||
token_url="https://idp.example/token", client_id="client", client_secret="secret",
|
||||
server_id="srv",
|
||||
name="srv",
|
||||
transport="http",
|
||||
auth_type="oauth2",
|
||||
token_url="https://idp.example/token",
|
||||
client_id="client",
|
||||
client_secret="secret",
|
||||
)
|
||||
|
||||
async def token_endpoint(url, form, headers):
|
||||
assert url == server.token_url and form["grant_type"] == "refresh_token"
|
||||
return {"access_token": "rotated", **({"scope": response_scope} if response_scope is not None else {})}
|
||||
return Ok({"access_token": "rotated", **({"scope": response_scope} if response_scope is not None else {})})
|
||||
|
||||
refresher = AuthorizationCodeRefresher(lambda _: server, token_endpoint, _persist_credential)
|
||||
refresher = AuthorizationCodeRefresher(
|
||||
lambda _: server, token_endpoint, _persist_credential, purge=_purge_credential
|
||||
)
|
||||
refreshed = await refresher.refresh("alice", "srv", OAuthToken("old", refresh_token="refresh", scopes=prior_scopes))
|
||||
assert refreshed is not None and refreshed.scopes == tuple(expected)
|
||||
payload = decode_oauth_payload(credential_database.upsert.call_args.kwargs["data"]["create"]["credential_b64"])
|
||||
|
|
@ -394,3 +413,128 @@ async def test_expired_unverified_credential_never_reaches_refresh(monkeypatch):
|
|||
store = LazyPerUserOAuthTokenStore(lambda _: server, redis_available=lambda: False)
|
||||
assert await store.fetch("alice", "srv") is None
|
||||
post.assert_not_awaited()
|
||||
|
||||
|
||||
async def _stored_row(credential_database, refresh_token):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import _persist_credential
|
||||
|
||||
await _persist_credential("alice", "srv", "expired-access", refresh_token, -60, ("read",))
|
||||
credential_b64 = credential_database.upsert.call_args.kwargs["data"]["create"]["credential_b64"]
|
||||
return SimpleNamespace(user_id="alice", server_id="srv", credential_b64=credential_b64)
|
||||
|
||||
|
||||
def _refreshable_server():
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
|
||||
return MCPServer(
|
||||
server_id="srv",
|
||||
name="srv",
|
||||
transport="http",
|
||||
auth_type="oauth2",
|
||||
oauth2_flow="authorization_code",
|
||||
token_url="https://idp.example/token",
|
||||
client_id="client",
|
||||
client_secret="secret",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error_code", ["invalid_grant", "invalid_token"])
|
||||
async def test_revoked_refresh_token_deletes_the_stored_credential(
|
||||
monkeypatch, respx_mock, credential_database, error_code
|
||||
):
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module
|
||||
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False))
|
||||
row = await _stored_row(credential_database, "revoked-refresh")
|
||||
credential_database.find_unique.return_value = row
|
||||
upstream = respx_mock.post("https://idp.example/token").respond(400, json={"error": error_code})
|
||||
|
||||
store = LazyPerUserOAuthTokenStore(lambda _: _refreshable_server(), redis_available=lambda: False)
|
||||
assert await store.fetch("alice", "srv") is None
|
||||
assert upstream.call_count == 1
|
||||
credential_database.delete_many.assert_awaited_once_with(
|
||||
where={"user_id": "alice", "server_id": "srv", "credential_b64": row.credential_b64}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_revoked_refresh_completes_when_redis_comes_up_mid_fetch(monkeypatch, respx_mock, credential_database):
|
||||
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module
|
||||
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False))
|
||||
credential_database.find_unique.return_value = await _stored_row(credential_database, "revoked-refresh")
|
||||
redis_available = _RedisAvailability()
|
||||
|
||||
def revoke_while_redis_comes_up(_request: httpx.Request) -> httpx.Response:
|
||||
redis_available.available = True
|
||||
return httpx.Response(400, json={"error": "invalid_grant"})
|
||||
|
||||
respx_mock.post("https://idp.example/token").mock(side_effect=revoke_while_redis_comes_up)
|
||||
store = LazyPerUserOAuthTokenStore(lambda _: _refreshable_server(), redis_available=redis_available)
|
||||
monkeypatch.setattr(global_mcp_server_manager, "_per_user_oauth_token_store", store)
|
||||
|
||||
assert await asyncio.wait_for(store.fetch("alice", "srv"), timeout=5) is None
|
||||
credential_database.delete_many.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
[
|
||||
{"status_code": 503, "json": {"error": "temporarily_unavailable"}},
|
||||
{"status_code": 400, "json": {"error": "invalid_client"}},
|
||||
{"status_code": 200, "text": "not json"},
|
||||
{"status_code": 200, "json": ["not", "an", "object"]},
|
||||
{"status_code": 400, "json": ["invalid_grant"]},
|
||||
{"status_code": 400, "text": "invalid_grant"},
|
||||
],
|
||||
)
|
||||
async def test_transient_refresh_failure_keeps_the_stored_credential_for_retry(
|
||||
monkeypatch, respx_mock, credential_database, response
|
||||
):
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module
|
||||
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False))
|
||||
credential_database.find_unique.return_value = await _stored_row(credential_database, "live-refresh")
|
||||
upstream = respx_mock.post("https://idp.example/token").respond(**response)
|
||||
|
||||
store = LazyPerUserOAuthTokenStore(lambda _: _refreshable_server(), redis_available=lambda: False)
|
||||
assert await store.fetch("alice", "srv") is None
|
||||
upstream.respond(200, json={"access_token": "recovered", "expires_in": 3600})
|
||||
recovered = await store.fetch("alice", "srv")
|
||||
assert recovered is not None and recovered.access_token == "recovered"
|
||||
assert upstream.call_count == 2
|
||||
credential_database.delete_many.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("transport_error", [httpx.ConnectError("refused"), httpx.ReadTimeout("slow")])
|
||||
async def test_unreachable_token_endpoint_keeps_the_stored_credential(
|
||||
monkeypatch, respx_mock, credential_database, transport_error
|
||||
):
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module
|
||||
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False))
|
||||
credential_database.find_unique.return_value = await _stored_row(credential_database, "live-refresh")
|
||||
respx_mock.post("https://idp.example/token").mock(side_effect=transport_error)
|
||||
|
||||
store = LazyPerUserOAuthTokenStore(lambda _: _refreshable_server(), redis_available=lambda: False)
|
||||
assert await store.fetch("alice", "srv") is None
|
||||
credential_database.delete_many.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_purge_skips_a_credential_already_replaced_by_reauthorization(monkeypatch, credential_database):
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import _purge_credential
|
||||
|
||||
credential_database.find_unique.return_value = await _stored_row(credential_database, "fresh-refresh")
|
||||
await _purge_credential("alice", "srv", "revoked-refresh")
|
||||
credential_database.delete_many.assert_not_awaited()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue