fix(mcp): derive MCP bridge and session keys with the FIPS provider's KBKDF (#50032)

* fix(mcp): derive MCP bridge and session keys with the FIPS provider's KBKDF

With OpenSSL enforcing FIPS, hashlib.scrypt raises, so every MCP session
token and bridge envelope request failed with a 500. Under FIPS the keys
now come from LITELLM_SALT_KEY through SP 800-108r1 KBKDF run by the
provider's EVP_KDF. Without FIPS the scrypt keys are unchanged

* fix(mcp): keep master key rotation invalidating MCP tokens under FIPS

Pass the master key as the KBKDF context, so rotating it changes the
three FIPS-mode keys as it changes the scrypt keys

* test(mcp): annotate new FIPS key test locals with Final

* fix(mcp): derive the FIPS MCP keys only when general_settings.fips_mode is on

On a FIPS OpenSSL 1.1.1 host (RHEL 8) scrypt still works, so keying the switch on
openssl_enforces_fips() sent those hosts to the OpenSSL 3 only KBKDF and broke MCP
sessions and bridge envelopes that worked on main. fips_mode_active() is the same
reader B1 adds, set by enforce_fips_mode at startup.

* refactor(fips): move the salt key KBKDF into encrypt_decrypt_utils

The KBKDF helper derives keys from LITELLM_SALT_KEY, so it now sits next to the other salt key helpers, and fips.py goes back to main's version.

* Pin the MCP key KBKDF fetch to the FIPS provider

Fetch KBKDF with the property query fips=yes, so a change to the process's default
properties after boot can't move the derivation to a non-FIPS provider. The shared
fips_openssl test fixture drops the pin, since the test OpenSSL has no FIPS provider.

litellm/proxy/common_utils/encrypt_decrypt_utils.py

@@ -1,11 +1,14 @@
 import base64
+import ctypes
 import os
+import re
 from collections.abc import Mapping
 from typing import Final, Literal, cast
 
 from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_proxy_logger
+from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR
 
 # Versioned ciphertext marker for AES-256-GCM values.
 # Format: "v2:gcm:" + base64url(nonce(12) || ciphertext || tag(16)).
@@ -42,6 +45,72 @@ def get_salt_key() -> str | None:
 
 _get_salt_key: Final = get_salt_key
 
+_SALT_KEY_HEX: Final = re.compile(r"(?:sk-)?((?:[0-9a-fA-F]{2}){32,})")
+_OSSL_PARAM_UTF8_STRING: Final = 4
+_OSSL_PARAM_OCTET_STRING: Final = 5
+_OSSL_PARAM_UNMODIFIED: Final = ctypes.c_size_t(-1).value
+_DERIVED_KEY_BYTES: Final = 32
+_KBKDF_PROPERTIES: Final = b"fips=yes"
+
+
+class _OsslParam(ctypes.Structure):
+    _fields_ = (
+        ("key", ctypes.c_char_p),
+        ("data_type", ctypes.c_uint),
+        ("data", ctypes.c_char_p),
+        ("data_size", ctypes.c_size_t),
+        ("return_size", ctypes.c_size_t),
+    )
+
+
+def _param(name: bytes, data_type: int, data: bytes) -> _OsslParam:
+    return _OsslParam(name, data_type, data, len(data), _OSSL_PARAM_UNMODIFIED)
+
+
+def _kbkdf_hmac_sha256(key: bytes, label: bytes, context: bytes) -> bytes:
+    import _hashlib
+
+    libcrypto: Final = ctypes.CDLL(_hashlib.__file__)
+    kdf_fetch: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p)(
+        ("EVP_KDF_fetch", libcrypto)
+    )
+    ctx_new: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(("EVP_KDF_CTX_new", libcrypto))
+    derive: Final = ctypes.CFUNCTYPE(
+        ctypes.c_int, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(_OsslParam)
+    )(("EVP_KDF_derive", libcrypto))
+    ctx_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_CTX_free", libcrypto))
+    kdf_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_free", libcrypto))
+    params: Final = (_OsslParam * 6)(
+        _param(b"mac", _OSSL_PARAM_UTF8_STRING, b"HMAC"),
+        _param(b"digest", _OSSL_PARAM_UTF8_STRING, b"SHA256"),
+        _param(b"key", _OSSL_PARAM_OCTET_STRING, key),
+        _param(b"salt", _OSSL_PARAM_OCTET_STRING, label),
+        _param(b"info", _OSSL_PARAM_OCTET_STRING, context),
+    )
+    derived: Final = ctypes.create_string_buffer(_DERIVED_KEY_BYTES)
+    kdf: Final = kdf_fetch(None, b"KBKDF", _KBKDF_PROPERTIES)
+    ctx: Final = ctx_new(kdf)
+    try:
+        if not ctx or derive(ctx, derived, _DERIVED_KEY_BYTES, params) != 1:
+            raise ValueError("OpenSSL could not run KBKDF")
+        return derived.raw
+    finally:
+        ctx_free(ctx)
+        kdf_free(kdf)
+
+
+def derive_key_from_salt_key(label: bytes, context: bytes) -> str:
+    """Hex of 32 bytes derived from LITELLM_SALT_KEY under label and context by SP 800-108r1 KBKDF in counter mode
+    with HMAC-SHA256, run by EVP_KDF from the FIPS provider of the OpenSSL that hashlib uses. Raises ValueError when
+    LITELLM_SALT_KEY is not at least 32 bytes of hex, optionally prefixed with sk-, or when no FIPS provider runs it."""
+    salt_key: Final = _SALT_KEY_HEX.fullmatch(os.getenv(SALT_KEY_ENV_VAR, ""))
+    if salt_key is None:
+        raise ValueError(
+            f"With general_settings.fips_mode on, {SALT_KEY_ENV_VAR} must be at least 32 bytes of hex, such as "
+            "sk-$(openssl rand -hex 32)"
+        )
+    return _kbkdf_hmac_sha256(bytes.fromhex(salt_key.group(1)), label, context).hex()
+
 
 def _get_encryption_algorithm() -> str:
     """

litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py

@@ -16,6 +16,8 @@ from typing import Final, Literal, TypeAlias
 
 from pydantic import ConfigDict, SecretStr
 
+from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key
+from litellm.proxy.common_utils.fips import fips_mode_active
 from litellm.proxy.mcp_server.outbound_credentials.envelope import (
     EnvelopeIdentity,
     EnvelopeKeys,
@@ -61,7 +63,15 @@ def envelope_keys_from_master_key(master_key: str) -> EnvelopeKeys:
     fixed for a process), so the KDF runs once per key and adds nothing to the per-request
     admission path. The derivation is deterministic; rotating ``master_key`` invalidates
     every outstanding envelope, which is the intended behavior for a signing-key change.
+
+    When general_settings.fips_mode is on, both keys come from :func:`derive_key_from_salt_key` with the master key as
+    context instead.
     """
+    if fips_mode_active():
+        return EnvelopeKeys(
+            signing_key=SecretStr(derive_key_from_salt_key(_SIGNING_KEY_DOMAIN, master_key.encode())),
+            encryption_key=SecretStr(derive_key_from_salt_key(_ENCRYPTION_KEY_DOMAIN, master_key.encode())),
+        )
     signing: Final = hashlib.scrypt(
         master_key.encode(),
         salt=_SIGNING_KEY_DOMAIN,

litellm/proxy/mcp_server/outbound_credentials/session_credentials.py

@@ -22,6 +22,8 @@ from typing import Final, Literal, TypeAlias
 
 from pydantic import ConfigDict, Field, SecretStr, ValidationError
 
+from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key
+from litellm.proxy.common_utils.fips import fips_mode_active
 from litellm.proxy.mcp_server.outbound_credentials.session_token import (
     AsymmetricSessionKeys,
     OpenedSessionToken,
@@ -59,7 +61,14 @@ def session_keys_from_master_key(master_key: str) -> SessionKeys:
     into the other. The result is cached (the master key is fixed for a process); rotating
     ``master_key`` invalidates every outstanding session, which is the intended behavior
     for a signing-key change.
+
+    When general_settings.fips_mode is on, the key comes from :func:`derive_key_from_salt_key` with the master key as
+    context instead.
     """
+    if fips_mode_active():
+        return SessionKeys(
+            signing_key=SecretStr(derive_key_from_salt_key(_SESSION_SIGNING_KEY_DOMAIN, master_key.encode()))
+        )
     signing: Final = hashlib.scrypt(
         master_key.encode(),
         salt=_SESSION_SIGNING_KEY_DOMAIN,

tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py

@@ -6,8 +6,11 @@ gate, and the backward-compatibility guarantees that let legacy XSalsa20-Poly130
 (nacl) ciphertext and new AES values coexist and decrypt correctly.
 """
 
+import _hashlib
 import base64
+import hmac
 import re
+from typing import Final
 
 import pytest
 
@@ -17,6 +20,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_bearer_token,
     decrypt_if_encrypted_with,
     decrypt_value_helper,
+    derive_key_from_salt_key,
     encrypt_bearer_token,
     encrypt_value,
     encrypt_value_helper,
@@ -266,3 +270,29 @@ def test_bearer_token_uses_only_header_safe_characters(length: int):
 
     assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token
     assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length
+
+
+_needs_shared_openssl: Final = pytest.mark.skipif(
+    not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach"
+)
+
+
+@_needs_shared_openssl
+@pytest.mark.usefixtures("fips_openssl")
+@pytest.mark.parametrize("salt_key", ["sk-" + bytes(range(32)).hex(), bytes(range(32)).hex().upper()])
+def test_derive_key_from_salt_key_is_sp800_108_counter_mode_hmac_sha256(
+    monkeypatch: pytest.MonkeyPatch, salt_key: str
+) -> None:
+    label: Final = b"litellm-test:label:"
+    context: Final = b"sk-test-master-key"
+    monkeypatch.setenv("LITELLM_SALT_KEY", salt_key)
+    fixed_input: Final = (1).to_bytes(4, "big") + label + b"\x00" + context + (256).to_bytes(4, "big")
+    assert derive_key_from_salt_key(label, context) == hmac.digest(bytes(range(32)), fixed_input, "sha256").hex()
+
+
+@_needs_shared_openssl
+@pytest.mark.skipif(_hashlib.get_fips_mode() == 1, reason="this OpenSSL runs a FIPS provider")
+def test_derive_key_from_salt_key_refuses_an_openssl_without_a_fips_provider(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + bytes(range(32)).hex())
+    with pytest.raises(ValueError, match="OpenSSL could not run KBKDF"):
+        derive_key_from_salt_key(b"litellm-test:label:", b"sk-test-master-key")

tests/unit/proxy/conftest.py

@@ -523,6 +523,7 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]:
 def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1)
     monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported")))
+    monkeypatch.setattr("litellm.proxy.common_utils.encrypt_decrypt_utils._KBKDF_PROPERTIES", None)
 
 
 @pytest.fixture

tests/unit/proxy/mcp_server/outbound_credentials/conftest.py

@@ -0,0 +1,20 @@
+from collections.abc import Iterator
+
+import pytest
+
+from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import envelope_keys_from_master_key
+from litellm.proxy.mcp_server.outbound_credentials.session_credentials import session_keys_from_master_key
+
+
+@pytest.fixture
+def openssl_enforcing_fips(fips_openssl: None) -> Iterator[None]:
+    envelope_keys_from_master_key.cache_clear()
+    session_keys_from_master_key.cache_clear()
+    yield
+    envelope_keys_from_master_key.cache_clear()
+    session_keys_from_master_key.cache_clear()
+
+
+@pytest.fixture
+def fips_mode_on(openssl_enforcing_fips: None, fips_mode: None) -> None:
+    return None

tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py

@@ -9,6 +9,7 @@ through the consumer; and no path leaks the upstream token in a repr.
 """
 
 from datetime import datetime, timedelta, timezone
+from typing import Final
 
 import pytest
 from pydantic import SecretStr
@@ -318,3 +319,33 @@ def test_is_bridge_envelope_shaped_rejects_non_envelope_bearer():
     assert is_bridge_envelope_shaped("Bearer sk-some-litellm-key") is False
     assert is_bridge_envelope_shaped("plain-upstream-token") is False
     assert is_bridge_envelope_shaped("") is False
+
+
+def test_envelope_keys_without_fips_stay_the_scrypt_keys_of_the_master_key() -> None:
+    keys: Final = envelope_keys_from_master_key(_MASTER_KEY)
+    assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == (
+        "34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed",
+        "eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250",
+    )
+
+
+@pytest.mark.usefixtures("openssl_enforcing_fips")
+def test_envelope_keys_on_a_fips_openssl_without_fips_mode_stay_the_scrypt_keys(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
+    keys: Final = envelope_keys_from_master_key(_MASTER_KEY)
+    assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == (
+        "34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed",
+        "eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250",
+    )
+
+
+@pytest.mark.usefixtures("fips_mode_on")
+@pytest.mark.parametrize("salt_key", ["", "sk-salt-change-me", _MASTER_KEY, "sk-" + "5a" * 31])
+def test_envelope_keys_under_fips_refuse_a_salt_key_that_is_not_32_bytes_of_hex(
+    monkeypatch: pytest.MonkeyPatch, salt_key: str
+) -> None:
+    monkeypatch.setenv("LITELLM_SALT_KEY", salt_key)
+    with pytest.raises(ValueError, match="LITELLM_SALT_KEY"):
+        envelope_keys_from_master_key(_MASTER_KEY)

tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py

@@ -1,6 +1,8 @@
 """Tests for the session-token KDF and the edge/token-endpoint resolvers."""
 
+import _hashlib
 from datetime import datetime, timedelta, timezone
+from typing import Final
 
 import pytest
 from cryptography.hazmat.primitives import serialization
@@ -230,3 +232,64 @@ def test_signing_config_error_detail_never_leaks_key_material():
     )
     assert isinstance(resolved, SessionSigningConfigError)
     assert pem.splitlines()[1] not in resolved.detail
+
+
+def test_session_key_without_fips_stays_the_scrypt_key_of_the_master_key() -> None:
+    keys: Final = session_keys_from_master_key(MASTER_KEY)
+    assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d"
+
+
+@pytest.mark.usefixtures("openssl_enforcing_fips")
+def test_session_key_on_a_fips_openssl_without_fips_mode_stays_the_scrypt_key(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
+    keys: Final = session_keys_from_master_key(MASTER_KEY)
+    assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d"
+
+
+def _fips_keys(master_key: str) -> tuple[str, str, str]:
+    envelope_keys_from_master_key.cache_clear()
+    session_keys_from_master_key.cache_clear()
+    envelope: Final = envelope_keys_from_master_key(master_key)
+    session: Final = session_keys_from_master_key(master_key)
+    return (
+        envelope.signing_key.get_secret_value(),
+        envelope.encryption_key.get_secret_value(),
+        session.signing_key.get_secret_value(),
+    )
+
+
+_needs_shared_openssl: Final = pytest.mark.skipif(
+    not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach"
+)
+
+
+@_needs_shared_openssl
+@pytest.mark.usefixtures("fips_mode_on")
+def test_keys_under_fips_come_from_the_salt_key_are_distinct_and_the_same_on_every_call(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32)
+    first: Final = _fips_keys(MASTER_KEY)
+    assert first == _fips_keys(MASTER_KEY)
+    assert len(set(first)) == 3, first
+    assert first == (
+        "ef694d415772aef09731c47709d4d9da47f7bbfcd6484836dea09c2bb387063b",
+        "d8a7cd70690fbf98c5581301b2e0460f2ad714f7c74b2af1b3dac3e0aa99db85",
+        "5e11e470bf9c838165b70d0b2b82a7ee5e2f00d9449eeb54f7c5bf9491b64354",
+    )
+
+
+@_needs_shared_openssl
+@pytest.mark.usefixtures("fips_mode_on")
+def test_keys_under_fips_all_change_when_the_master_key_rotates(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32)
+    before: Final = _fips_keys(MASTER_KEY)
+    after: Final = _fips_keys("sk-a-different-master-key")
+    assert set(before).isdisjoint(after), (before, after)
+
+
+@pytest.mark.usefixtures("fips_mode_on")
+def test_session_key_under_fips_refuses_an_unset_salt_key(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
+    with pytest.raises(ValueError, match="LITELLM_SALT_KEY"):
+        session_keys_from_master_key(MASTER_KEY)