diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 0666cc7e6c0..80bddbf0e66 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1900,6 +1900,26 @@ async def get_user_oauth_credential( return _parse_oauth_payload(decoded) +async def delete_user_oauth_credential_holding_refresh_token( + prisma_client: PrismaClient, + user_id: str, + server_id: str, + refresh_token: str, +) -> bool: + """Delete the user's OAuth credential only while it still holds ``refresh_token``. The delete is + keyed on the exact ciphertext read, so a re-authorization that lands concurrently survives.""" + row: Final = await _db_find_user_credential_row(prisma_client, user_id, server_id) + if row is None: + return False + payload: Final = decode_oauth_payload(row.credential_b64) + if payload is None or payload.get("refresh_token") != refresh_token: + return False + deleted: Final = await _user_credential_actions(prisma_client).delete_many( + where={"user_id": user_id, "server_id": server_id, "credential_b64": row.credential_b64} + ) + return deleted > 0 + + def _server_user_credential_item( row: "prisma_db_models.LiteLLM_MCPUserCredentials", ) -> MCPServerUserCredentialListItem: diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py index a653ef5e5ff..93330055748 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py @@ -11,10 +11,11 @@ across replicas - is the wrapping store's distributed single-flight, not this re from __future__ import annotations import time -from collections.abc import Awaitable, Callable -from typing import TYPE_CHECKING, Final, Protocol +from collections.abc import Awaitable, Callable, Mapping +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias from fastapi import HTTPException +from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import ( @@ -29,12 +30,29 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oa from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Result +from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import ( + RefreshFailure, +) if TYPE_CHECKING: from litellm.types.mcp_server.mcp_server_manager import MCPServer ServerLookup = Callable[[str], "MCPServer | None"] -TokenEndpointPost = Callable[[str, dict[str, str], dict[str, str]], Awaitable["dict[str, object] | None"]] +TokenEndpointPost: TypeAlias = Callable[ + [str, dict[str, str], dict[str, str]], Awaitable[Result[Mapping[str, object], RefreshFailure]] +] +CredentialPurge: TypeAlias = Callable[[str, str, str], Awaitable[None]] + +_REVOKED_GRANT_ERRORS: Final = frozenset({"invalid_grant", "invalid_token"}) + + +def classify_token_endpoint_refusal(status_code: int, error: str | None) -> RefreshFailure: + if 400 <= status_code < 500 and error in _REVOKED_GRANT_ERRORS: + return RefreshFailure.of_rejected(f"the token endpoint answered {status_code} {error}") + return RefreshFailure.of_unavailable( + f"the token endpoint answered {status_code} {error or 'without an error code'}" + ) class CredentialPersist(Protocol): @@ -71,13 +89,15 @@ def _parse_scopes(raw: object) -> tuple[str, ...] | None: class AuthorizationCodeRefresher: """``TokenRefresher`` for authorization_code: refresh_token grant against the server, then persist. - ``token_endpoint`` POSTs the OAuth form and returns the parsed JSON body (``None`` on any - transport/HTTP failure, mirroring v1: a failed refresh is a miss, not a 500). ``persist`` writes - the rotated triple for ``(user, server)`` - the v1 ``store_user_oauth_credential`` write, which - stays. Returns ``None`` (the arm challenges) when there is no refresh_token, the server lacks a - token endpoint, or the grant fails; never a stale or partial token. A rotated refresh_token from - the response replaces the old one; an omitted one is carried forward, as are the recorded scopes - when the response omits ``scope``. + ``token_endpoint`` POSTs the OAuth form and returns the parsed JSON body or a classified + ``RefreshFailure``. ``persist`` writes the rotated triple for ``(user, server)`` - the v1 + ``store_user_oauth_credential`` write, which stays. A ``rejected`` failure (the IdP answered + ``invalid_grant`` / ``invalid_token``) means the stored refresh token is dead, so ``purge`` drops + the credential that still holds it and the user is challenged to re-authorize; an ``unavailable`` + one keeps the credential so the next request retries. Returns ``None`` (the arm challenges) when + there is no refresh_token, the server lacks a token endpoint, or the grant fails; never a stale or + partial token. A rotated refresh_token from the response replaces the old one; an omitted one is + carried forward, as are the recorded scopes when the response omits ``scope``. """ def __init__( @@ -86,12 +106,14 @@ class AuthorizationCodeRefresher: token_endpoint: TokenEndpointPost, persist: CredentialPersist, *, + purge: CredentialPurge, clock: Callable[[], float] = time.time, identity_validator: BindingValidator = enforce_oauth_identity_binding, ) -> None: self._server_lookup = server_lookup self._token_endpoint = token_endpoint self._persist = persist + self._purge = purge self._clock = clock self._identity_validator = identity_validator @@ -139,9 +161,11 @@ class AuthorizationCodeRefresher: "refresh_token": token.refresh_token, **token_request.body, } - body: Final = await self._token_endpoint(token_url, form, token_request.headers) - if body is None: + outcome: Final = await self._token_endpoint(token_url, form, token_request.headers) + if isinstance(outcome, Error): + await self._on_failure(user_id, server_id, token.refresh_token, outcome.error) return None + body: Final = outcome.ok access_token: Final = body.get("access_token") if not isinstance(access_token, str) or not access_token: return None @@ -177,3 +201,24 @@ class AuthorizationCodeRefresher: identity_binding_proof=binding_proof, cimd_client_id=cimd_client_id, ) + + async def _on_failure(self, user_id: str, server_id: str, refresh_token: str, failure: RefreshFailure) -> None: + match failure.kind: + case "rejected": + verbose_logger.warning( + "MCP OAuth refresh rejected for user=%s server=%s (%s); removing the stored credential so " + "the user re-authorizes", + user_id, + server_id, + failure.detail, + ) + await self._purge(user_id, server_id, refresh_token) + case "unavailable": + verbose_logger.warning( + "MCP OAuth refresh failed for user=%s server=%s (%s); keeping the stored credential", + user_id, + server_id, + failure.detail, + ) + case _: + assert_never(failure.kind) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py index 2e9f3232c8d..04e11b061ae 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py @@ -15,10 +15,14 @@ from collections.abc import Callable, Mapping from functools import partial from typing import TYPE_CHECKING, Final +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.oauth_identity_binding import credential_binding_matches from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import ( AuthorizationCodeRefresher, + classify_token_endpoint_refusal, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.dual_cache_token_backend import ( AsyncCache, @@ -33,9 +37,13 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_sto TokenCacheBackend, TokenStoreUnavailable, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok, Result from litellm.proxy._experimental.mcp_server.outbound_credentials.runtime_refresh_coordinator import ( runtime_refresh_coordinator, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import ( + RefreshFailure, +) from litellm.proxy._experimental.mcp_server.outbound_credentials.token_cache_codec import ( OAuthTokenCacheCodec, ) @@ -95,7 +103,25 @@ async def _persist_credential( ) -async def _post_token_endpoint(url: str, form: dict[str, str], headers: dict[str, str]) -> dict[str, object] | None: +async def _purge_credential(user_id: str, server_id: str, refresh_token: str) -> None: + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 # avoids a proxy_server import cycle + delete_user_oauth_credential_holding_refresh_token, + ) + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 # avoids a proxy_server import cycle + + if prisma_client is None: + return + try: + await delete_user_oauth_credential_holding_refresh_token(prisma_client, user_id, server_id, refresh_token) + except Exception as exc: # noqa: BLE001 # the refresh already failed closed; the next refusal retries the purge + verbose_logger.warning( + "Failed to remove revoked MCP OAuth credential for user=%s server=%s: %s", user_id, server_id, exc + ) + + +async def _post_token_endpoint( + url: str, form: dict[str, str], headers: dict[str, str] +) -> Result[Mapping[str, object], RefreshFailure]: from litellm.llms.custom_httpx.http_handler import ( # noqa: PLC0415 get_async_httpx_client, # pyright: ignore ) @@ -105,17 +131,35 @@ async def _post_token_endpoint(url: str, form: dict[str, str], headers: dict[str # object and the refresher validates each field, so the untyped boundary is contained here. provider: Final = httpxSpecialProvider.Oauth2Check request_headers: Final = {"Accept": "application/json", **headers} - # A failed refresh is a miss, not a 500 (matches v1), so any error becomes None. try: client: Final = get_async_httpx_client(llm_provider=provider) # pyright: ignore response: Final = await client.post(url, headers=request_headers, data=form) # pyright: ignore response.raise_for_status() # pyright: ignore - body: Final[dict[str, object]] = response.json() # pyright: ignore + content: Final[bytes] = response.content # pyright: ignore + except httpx.HTTPStatusError as exc: + return Error(classify_token_endpoint_refusal(exc.response.status_code, _token_error_code(exc.response.content))) except Exception as exc: # noqa: BLE001 - verbose_logger.warning("MCP OAuth refresh request failed: %s", exc) + return Error(RefreshFailure.of_unavailable(f"the token endpoint request failed ({type(exc).__name__})")) + try: + return Ok(_TOKEN_RESPONSE.validate_json(content)) + except ValidationError: + return Error(RefreshFailure.of_unavailable("the token endpoint returned a malformed response")) + + +class _TokenErrorBody(BaseModel): + model_config = ConfigDict(frozen=True) + + error: str | None = None + + +_TOKEN_RESPONSE: Final = TypeAdapter(Mapping[str, object]) + + +def _token_error_code(content: bytes) -> str | None: + try: + return _TokenErrorBody.model_validate_json(content).error + except ValidationError: return None - else: - return body # pyright: ignore def _redis_cache_is_available() -> bool: @@ -164,7 +208,9 @@ def _build_per_user_oauth_token_store( server_lookup: ServerLookup, ) -> tuple[CachedOAuthTokenStore, bool]: backend, coordinator, uses_redis = _runtime_backend_and_coordinator() - refresher: Final = AuthorizationCodeRefresher(server_lookup, _post_token_endpoint, _persist_credential) + refresher: Final = AuthorizationCodeRefresher( + server_lookup, _post_token_endpoint, _persist_credential, purge=_purge_credential + ) refreshing: Final = RefreshingTokenStore( V2PerUserTokenStore(partial(_read_bound_credential, server_lookup)), refresher, coordinator=coordinator ) diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py b/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py index 3068e5a0116..f6425c0f39e 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py +++ b/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py @@ -4,10 +4,15 @@ import pytest from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import ( AuthorizationCodeRefresher, + classify_token_endpoint_refusal, ) from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) +from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Error, Ok +from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_refresher import ( + RefreshFailure, +) class _Server: @@ -45,11 +50,27 @@ def _endpoint(body, sink=None): async def post(url, form, headers): if sink is not None: sink.append((url, form, headers)) - return body + if body is None: + return Error(RefreshFailure.of_unavailable("token endpoint down")) + return Ok(body) return post +def _failing_endpoint(failure): + async def post(url, form, headers): + return Error(failure) + + return post + + +def _recording_purge(sink): + async def purge(user_id, server_id, refresh_token): + sink.append((user_id, server_id, refresh_token)) + + return purge + + def _recording_persist(sink): async def persist( user_id, server_id, access_token, refresh_token, expires_in, scopes, cimd_client_id=None @@ -62,12 +83,13 @@ def _recording_persist(sink): def _refresher( - server=None, body=None, *, post_sink=None, persist_sink=None, clock=lambda: 1000.0 + server=None, body=None, *, post_sink=None, persist_sink=None, purge_sink=None, clock=lambda: 1000.0 ): return AuthorizationCodeRefresher( _lookup(server if server is not None else _Server()), _endpoint(body, post_sink), _recording_persist(persist_sink if persist_sink is not None else []), + purge=_recording_purge(purge_sink if purge_sink is not None else []), clock=clock, ) @@ -184,6 +206,55 @@ async def test_grant_failure_does_not_persist(): assert persisted == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("error_code", ["invalid_grant", "invalid_token"]) +@pytest.mark.parametrize("status_code", [400, 401]) +async def test_revoked_refresh_token_purges_the_credential_that_holds_it(status_code, error_code): + persisted = [] + purged = [] + refresher = AuthorizationCodeRefresher( + _lookup(_Server()), + _failing_endpoint(classify_token_endpoint_refusal(status_code, error_code)), + _recording_persist(persisted), + purge=_recording_purge(purged), + ) + assert await refresher.refresh("alice", "srv", OAuthToken("old", refresh_token="revoked-rt")) is None + assert purged == [("alice", "srv", "revoked-rt")] + assert persisted == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "status_code, error_code", + [ + (503, "temporarily_unavailable"), + (500, "invalid_grant"), + (429, "slow_down"), + (400, "invalid_client"), + (400, "invalid_request"), + (400, None), + ], +) +async def test_transient_or_non_grant_refusal_keeps_the_credential(status_code, error_code): + purged = [] + refresher = AuthorizationCodeRefresher( + _lookup(_Server()), + _failing_endpoint(classify_token_endpoint_refusal(status_code, error_code)), + _recording_persist([]), + purge=_recording_purge(purged), + ) + assert await refresher.refresh("alice", "srv", OAuthToken("old", refresh_token="rt")) is None + assert purged == [] + + +@pytest.mark.asyncio +async def test_unreachable_token_endpoint_keeps_the_credential(): + purged = [] + refresher = _refresher(body=None, purge_sink=purged) + assert await refresher.refresh("alice", "srv", OAuthToken("old", refresh_token="rt")) is None + assert purged == [] + + @pytest.mark.asyncio async def test_response_without_access_token_does_not_persist(): persisted = [] @@ -304,6 +375,7 @@ async def test_identity_rejection_never_persists_or_returns_refreshed_token(): _lookup(_Server()), _endpoint({"access_token": "foreign-token", "id_token": "foreign-identity"}), persist, + purge=_recording_purge([]), identity_validator=validator, ) assert await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) is None @@ -320,6 +392,7 @@ async def test_verified_refresh_preserves_binding_proof_in_storage(): _lookup(_Server()), _endpoint({"access_token": "new", "refresh_token": "rotated"}), persist, + purge=_recording_purge([]), identity_validator=validator, ) token = await refresher.refresh("alice", "srv", OAuthToken(access_token="old", refresh_token="old-rt")) @@ -394,7 +467,7 @@ async def test_saved_cimd_identity_refreshes_without_discovery(monkeypatch): assert form["client_id"] == identity assert "client_secret" not in form assert "Authorization" not in headers - return {"access_token": "new", "refresh_token": "rotated"} + return Ok({"access_token": "new", "refresh_token": "rotated"}) server = MCPServer( server_id="srv", name="srv", transport="http", auth_type="oauth2", oauth2_flow="authorization_code", @@ -402,7 +475,9 @@ async def test_saved_cimd_identity_refreshes_without_discovery(monkeypatch): ) token = await V2PerUserTokenStore(read_credential).fetch("alice", "srv") assert token is not None - refreshed = await AuthorizationCodeRefresher(lambda _: server, post, persist).refresh("alice", "srv", token) + refreshed = await AuthorizationCodeRefresher( + lambda _: server, post, persist, purge=_recording_purge([]) + ).refresh("alice", "srv", token) assert refreshed is not None assert refreshed.access_token == "new" assert refreshed.refresh_token == "rotated" diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py b/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py index 5e2bb0f3722..2058ab82b7f 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py +++ b/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_per_user_oauth_store.py @@ -1,5 +1,6 @@ import asyncio +import httpx import pytest from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( @@ -35,6 +36,8 @@ def credential_database(monkeypatch): prisma = MagicMock() table = prisma.db.litellm_mcpusercredentials table.upsert = AsyncMock() + table.find_unique = AsyncMock(return_value=None) + table.delete_many = AsyncMock(return_value=1) monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setenv("LITELLM_SALT_KEY", "credential-persistence-test-salt") return table @@ -59,23 +62,39 @@ async def test_credential_persistence_preserves_known_and_unknown_scopes(credent [((), None, []), (("read", "write"), "", []), (("read", "write"), None, ["read", "write"])], ) async def test_native_refresh_persists_actual_grant_through_storage_adapter( - credential_database, prior_scopes, response_scope, expected, + credential_database, + prior_scopes, + response_scope, + expected, ): from litellm.proxy._experimental.mcp_server.db import decode_oauth_payload - from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import AuthorizationCodeRefresher - from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import _persist_credential + from litellm.proxy._experimental.mcp_server.outbound_credentials.authz_code_refresher import ( + AuthorizationCodeRefresher, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import ( + _persist_credential, + _purge_credential, + ) + from litellm.proxy._experimental.mcp_server.outbound_credentials.result import Ok from litellm.types.mcp_server.mcp_server_manager import MCPServer server = MCPServer( - server_id="srv", name="srv", transport="http", auth_type="oauth2", - token_url="https://idp.example/token", client_id="client", client_secret="secret", + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + token_url="https://idp.example/token", + client_id="client", + client_secret="secret", ) async def token_endpoint(url, form, headers): assert url == server.token_url and form["grant_type"] == "refresh_token" - return {"access_token": "rotated", **({"scope": response_scope} if response_scope is not None else {})} + return Ok({"access_token": "rotated", **({"scope": response_scope} if response_scope is not None else {})}) - refresher = AuthorizationCodeRefresher(lambda _: server, token_endpoint, _persist_credential) + refresher = AuthorizationCodeRefresher( + lambda _: server, token_endpoint, _persist_credential, purge=_purge_credential + ) refreshed = await refresher.refresh("alice", "srv", OAuthToken("old", refresh_token="refresh", scopes=prior_scopes)) assert refreshed is not None and refreshed.scopes == tuple(expected) payload = decode_oauth_payload(credential_database.upsert.call_args.kwargs["data"]["create"]["credential_b64"]) @@ -394,3 +413,128 @@ async def test_expired_unverified_credential_never_reaches_refresh(monkeypatch): store = LazyPerUserOAuthTokenStore(lambda _: server, redis_available=lambda: False) assert await store.fetch("alice", "srv") is None post.assert_not_awaited() + + +async def _stored_row(credential_database, refresh_token): + from types import SimpleNamespace + + from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import _persist_credential + + await _persist_credential("alice", "srv", "expired-access", refresh_token, -60, ("read",)) + credential_b64 = credential_database.upsert.call_args.kwargs["data"]["create"]["credential_b64"] + return SimpleNamespace(user_id="alice", server_id="srv", credential_b64=credential_b64) + + +def _refreshable_server(): + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + return MCPServer( + server_id="srv", + name="srv", + transport="http", + auth_type="oauth2", + oauth2_flow="authorization_code", + token_url="https://idp.example/token", + client_id="client", + client_secret="secret", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("error_code", ["invalid_grant", "invalid_token"]) +async def test_revoked_refresh_token_deletes_the_stored_credential( + monkeypatch, respx_mock, credential_database, error_code +): + from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False)) + row = await _stored_row(credential_database, "revoked-refresh") + credential_database.find_unique.return_value = row + upstream = respx_mock.post("https://idp.example/token").respond(400, json={"error": error_code}) + + store = LazyPerUserOAuthTokenStore(lambda _: _refreshable_server(), redis_available=lambda: False) + assert await store.fetch("alice", "srv") is None + assert upstream.call_count == 1 + credential_database.delete_many.assert_awaited_once_with( + where={"user_id": "alice", "server_id": "srv", "credential_b64": row.credential_b64} + ) + + +@pytest.mark.asyncio +async def test_revoked_refresh_completes_when_redis_comes_up_mid_fetch(monkeypatch, respx_mock, credential_database): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False)) + credential_database.find_unique.return_value = await _stored_row(credential_database, "revoked-refresh") + redis_available = _RedisAvailability() + + def revoke_while_redis_comes_up(_request: httpx.Request) -> httpx.Response: + redis_available.available = True + return httpx.Response(400, json={"error": "invalid_grant"}) + + respx_mock.post("https://idp.example/token").mock(side_effect=revoke_while_redis_comes_up) + store = LazyPerUserOAuthTokenStore(lambda _: _refreshable_server(), redis_available=redis_available) + monkeypatch.setattr(global_mcp_server_manager, "_per_user_oauth_token_store", store) + + assert await asyncio.wait_for(store.fetch("alice", "srv"), timeout=5) is None + credential_database.delete_many.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response", + [ + {"status_code": 503, "json": {"error": "temporarily_unavailable"}}, + {"status_code": 400, "json": {"error": "invalid_client"}}, + {"status_code": 200, "text": "not json"}, + {"status_code": 200, "json": ["not", "an", "object"]}, + {"status_code": 400, "json": ["invalid_grant"]}, + {"status_code": 400, "text": "invalid_grant"}, + ], +) +async def test_transient_refresh_failure_keeps_the_stored_credential_for_retry( + monkeypatch, respx_mock, credential_database, response +): + from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False)) + credential_database.find_unique.return_value = await _stored_row(credential_database, "live-refresh") + upstream = respx_mock.post("https://idp.example/token").respond(**response) + + store = LazyPerUserOAuthTokenStore(lambda _: _refreshable_server(), redis_available=lambda: False) + assert await store.fetch("alice", "srv") is None + upstream.respond(200, json={"access_token": "recovered", "expires_in": 3600}) + recovered = await store.fetch("alice", "srv") + assert recovered is not None and recovered.access_token == "recovered" + assert upstream.call_count == 2 + credential_database.delete_many.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transport_error", [httpx.ConnectError("refused"), httpx.ReadTimeout("slow")]) +async def test_unreachable_token_endpoint_keeps_the_stored_credential( + monkeypatch, respx_mock, credential_database, transport_error +): + from litellm.proxy._experimental.mcp_server.outbound_credentials import per_user_oauth_store as module + + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setattr(module, "_runtime_backend_and_coordinator", lambda: (None, None, False)) + credential_database.find_unique.return_value = await _stored_row(credential_database, "live-refresh") + respx_mock.post("https://idp.example/token").mock(side_effect=transport_error) + + store = LazyPerUserOAuthTokenStore(lambda _: _refreshable_server(), redis_available=lambda: False) + assert await store.fetch("alice", "srv") is None + credential_database.delete_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_purge_skips_a_credential_already_replaced_by_reauthorization(monkeypatch, credential_database): + from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import _purge_credential + + credential_database.find_unique.return_value = await _stored_row(credential_database, "fresh-refresh") + await _purge_credential("alice", "srv", "revoked-refresh") + credential_database.delete_many.assert_not_awaited()