From b603e84c020cbfb554d4857dd81bde0a0572e0dc Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 7 Nov 2026 13:33:27 -0800 Subject: [PATCH] fix(mcp): derive MCP bridge and session keys with the FIPS provider's KBKDF (#50032) * fix(mcp): derive MCP bridge and session keys with the FIPS provider's KBKDF With OpenSSL enforcing FIPS, hashlib.scrypt raises, so every MCP session token and bridge envelope request failed with a 500. Under FIPS the keys now come from LITELLM_SALT_KEY through SP 800-108r1 KBKDF run by the provider's EVP_KDF. Without FIPS the scrypt keys are unchanged * fix(mcp): keep master key rotation invalidating MCP tokens under FIPS Pass the master key as the KBKDF context, so rotating it changes the three FIPS-mode keys as it changes the scrypt keys * test(mcp): annotate new FIPS key test locals with Final * fix(mcp): derive the FIPS MCP keys only when general_settings.fips_mode is on On a FIPS OpenSSL 1.1.1 host (RHEL 8) scrypt still works, so keying the switch on openssl_enforces_fips() sent those hosts to the OpenSSL 3 only KBKDF and broke MCP sessions and bridge envelopes that worked on main. fips_mode_active() is the same reader B1 adds, set by enforce_fips_mode at startup. * refactor(fips): move the salt key KBKDF into encrypt_decrypt_utils The KBKDF helper derives keys from LITELLM_SALT_KEY, so it now sits next to the other salt key helpers, and fips.py goes back to main's version. * Pin the MCP key KBKDF fetch to the FIPS provider Fetch KBKDF with the property query fips=yes, so a change to the process's default properties after boot can't move the derivation to a non-FIPS provider. The shared fips_openssl test fixture drops the pin, since the test OpenSSL has no FIPS provider. --- .../common_utils/encrypt_decrypt_utils.py | 69 +++++++++++++++++++ .../bridge_credentials.py | 10 +++ .../session_credentials.py | 9 +++ .../test_encrypt_decrypt_utils.py | 30 ++++++++ tests/unit/proxy/conftest.py | 1 + .../outbound_credentials/conftest.py | 20 ++++++ .../test_bridge_credentials.py | 31 +++++++++ .../test_session_credentials.py | 63 +++++++++++++++++ 8 files changed, 233 insertions(+) create mode 100644 tests/unit/proxy/mcp_server/outbound_credentials/conftest.py diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index e5f6bc5dda8..644a35ef4ef 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -1,11 +1,14 @@ import base64 +import ctypes import os +import re from collections.abc import Mapping from typing import Final, Literal, cast from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR # Versioned ciphertext marker for AES-256-GCM values. # Format: "v2:gcm:" + base64url(nonce(12) || ciphertext || tag(16)). @@ -42,6 +45,72 @@ def get_salt_key() -> str | None: _get_salt_key: Final = get_salt_key +_SALT_KEY_HEX: Final = re.compile(r"(?:sk-)?((?:[0-9a-fA-F]{2}){32,})") +_OSSL_PARAM_UTF8_STRING: Final = 4 +_OSSL_PARAM_OCTET_STRING: Final = 5 +_OSSL_PARAM_UNMODIFIED: Final = ctypes.c_size_t(-1).value +_DERIVED_KEY_BYTES: Final = 32 +_KBKDF_PROPERTIES: Final = b"fips=yes" + + +class _OsslParam(ctypes.Structure): + _fields_ = ( + ("key", ctypes.c_char_p), + ("data_type", ctypes.c_uint), + ("data", ctypes.c_char_p), + ("data_size", ctypes.c_size_t), + ("return_size", ctypes.c_size_t), + ) + + +def _param(name: bytes, data_type: int, data: bytes) -> _OsslParam: + return _OsslParam(name, data_type, data, len(data), _OSSL_PARAM_UNMODIFIED) + + +def _kbkdf_hmac_sha256(key: bytes, label: bytes, context: bytes) -> bytes: + import _hashlib + + libcrypto: Final = ctypes.CDLL(_hashlib.__file__) + kdf_fetch: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p)( + ("EVP_KDF_fetch", libcrypto) + ) + ctx_new: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(("EVP_KDF_CTX_new", libcrypto)) + derive: Final = ctypes.CFUNCTYPE( + ctypes.c_int, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(_OsslParam) + )(("EVP_KDF_derive", libcrypto)) + ctx_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_CTX_free", libcrypto)) + kdf_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_free", libcrypto)) + params: Final = (_OsslParam * 6)( + _param(b"mac", _OSSL_PARAM_UTF8_STRING, b"HMAC"), + _param(b"digest", _OSSL_PARAM_UTF8_STRING, b"SHA256"), + _param(b"key", _OSSL_PARAM_OCTET_STRING, key), + _param(b"salt", _OSSL_PARAM_OCTET_STRING, label), + _param(b"info", _OSSL_PARAM_OCTET_STRING, context), + ) + derived: Final = ctypes.create_string_buffer(_DERIVED_KEY_BYTES) + kdf: Final = kdf_fetch(None, b"KBKDF", _KBKDF_PROPERTIES) + ctx: Final = ctx_new(kdf) + try: + if not ctx or derive(ctx, derived, _DERIVED_KEY_BYTES, params) != 1: + raise ValueError("OpenSSL could not run KBKDF") + return derived.raw + finally: + ctx_free(ctx) + kdf_free(kdf) + + +def derive_key_from_salt_key(label: bytes, context: bytes) -> str: + """Hex of 32 bytes derived from LITELLM_SALT_KEY under label and context by SP 800-108r1 KBKDF in counter mode + with HMAC-SHA256, run by EVP_KDF from the FIPS provider of the OpenSSL that hashlib uses. Raises ValueError when + LITELLM_SALT_KEY is not at least 32 bytes of hex, optionally prefixed with sk-, or when no FIPS provider runs it.""" + salt_key: Final = _SALT_KEY_HEX.fullmatch(os.getenv(SALT_KEY_ENV_VAR, "")) + if salt_key is None: + raise ValueError( + f"With general_settings.fips_mode on, {SALT_KEY_ENV_VAR} must be at least 32 bytes of hex, such as " + "sk-$(openssl rand -hex 32)" + ) + return _kbkdf_hmac_sha256(bytes.fromhex(salt_key.group(1)), label, context).hex() + def _get_encryption_algorithm() -> str: """ diff --git a/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py index 2326bcebdc3..cc3e99263c9 100644 --- a/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py @@ -16,6 +16,8 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, SecretStr +from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.mcp_server.outbound_credentials.envelope import ( EnvelopeIdentity, EnvelopeKeys, @@ -61,7 +63,15 @@ def envelope_keys_from_master_key(master_key: str) -> EnvelopeKeys: fixed for a process), so the KDF runs once per key and adds nothing to the per-request admission path. The derivation is deterministic; rotating ``master_key`` invalidates every outstanding envelope, which is the intended behavior for a signing-key change. + + When general_settings.fips_mode is on, both keys come from :func:`derive_key_from_salt_key` with the master key as + context instead. """ + if fips_mode_active(): + return EnvelopeKeys( + signing_key=SecretStr(derive_key_from_salt_key(_SIGNING_KEY_DOMAIN, master_key.encode())), + encryption_key=SecretStr(derive_key_from_salt_key(_ENCRYPTION_KEY_DOMAIN, master_key.encode())), + ) signing: Final = hashlib.scrypt( master_key.encode(), salt=_SIGNING_KEY_DOMAIN, diff --git a/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py index 6e3d09f4dd2..7cb89e53cdb 100644 --- a/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py @@ -22,6 +22,8 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, Field, SecretStr, ValidationError +from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.mcp_server.outbound_credentials.session_token import ( AsymmetricSessionKeys, OpenedSessionToken, @@ -59,7 +61,14 @@ def session_keys_from_master_key(master_key: str) -> SessionKeys: into the other. The result is cached (the master key is fixed for a process); rotating ``master_key`` invalidates every outstanding session, which is the intended behavior for a signing-key change. + + When general_settings.fips_mode is on, the key comes from :func:`derive_key_from_salt_key` with the master key as + context instead. """ + if fips_mode_active(): + return SessionKeys( + signing_key=SecretStr(derive_key_from_salt_key(_SESSION_SIGNING_KEY_DOMAIN, master_key.encode())) + ) signing: Final = hashlib.scrypt( master_key.encode(), salt=_SESSION_SIGNING_KEY_DOMAIN, diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index 1163090c510..5b0d5f014f8 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -6,8 +6,11 @@ gate, and the backward-compatibility guarantees that let legacy XSalsa20-Poly130 (nacl) ciphertext and new AES values coexist and decrypt correctly. """ +import _hashlib import base64 +import hmac import re +from typing import Final import pytest @@ -17,6 +20,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_bearer_token, decrypt_if_encrypted_with, decrypt_value_helper, + derive_key_from_salt_key, encrypt_bearer_token, encrypt_value, encrypt_value_helper, @@ -266,3 +270,29 @@ def test_bearer_token_uses_only_header_safe_characters(length: int): assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length + + +_needs_shared_openssl: Final = pytest.mark.skipif( + not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach" +) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.parametrize("salt_key", ["sk-" + bytes(range(32)).hex(), bytes(range(32)).hex().upper()]) +def test_derive_key_from_salt_key_is_sp800_108_counter_mode_hmac_sha256( + monkeypatch: pytest.MonkeyPatch, salt_key: str +) -> None: + label: Final = b"litellm-test:label:" + context: Final = b"sk-test-master-key" + monkeypatch.setenv("LITELLM_SALT_KEY", salt_key) + fixed_input: Final = (1).to_bytes(4, "big") + label + b"\x00" + context + (256).to_bytes(4, "big") + assert derive_key_from_salt_key(label, context) == hmac.digest(bytes(range(32)), fixed_input, "sha256").hex() + + +@_needs_shared_openssl +@pytest.mark.skipif(_hashlib.get_fips_mode() == 1, reason="this OpenSSL runs a FIPS provider") +def test_derive_key_from_salt_key_refuses_an_openssl_without_a_fips_provider(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + bytes(range(32)).hex()) + with pytest.raises(ValueError, match="OpenSSL could not run KBKDF"): + derive_key_from_salt_key(b"litellm-test:label:", b"sk-test-master-key") diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index d08a98242ef..8d8781be0a8 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -523,6 +523,7 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]: def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported"))) + monkeypatch.setattr("litellm.proxy.common_utils.encrypt_decrypt_utils._KBKDF_PROPERTIES", None) @pytest.fixture diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py b/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py new file mode 100644 index 00000000000..bd80cba1c62 --- /dev/null +++ b/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py @@ -0,0 +1,20 @@ +from collections.abc import Iterator + +import pytest + +from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import envelope_keys_from_master_key +from litellm.proxy.mcp_server.outbound_credentials.session_credentials import session_keys_from_master_key + + +@pytest.fixture +def openssl_enforcing_fips(fips_openssl: None) -> Iterator[None]: + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + yield + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + + +@pytest.fixture +def fips_mode_on(openssl_enforcing_fips: None, fips_mode: None) -> None: + return None diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py index 47b6c3c2fa9..0ba19d13955 100644 --- a/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py @@ -9,6 +9,7 @@ through the consumer; and no path leaks the upstream token in a repr. """ from datetime import datetime, timedelta, timezone +from typing import Final import pytest from pydantic import SecretStr @@ -318,3 +319,33 @@ def test_is_bridge_envelope_shaped_rejects_non_envelope_bearer(): assert is_bridge_envelope_shaped("Bearer sk-some-litellm-key") is False assert is_bridge_envelope_shaped("plain-upstream-token") is False assert is_bridge_envelope_shaped("") is False + + +def test_envelope_keys_without_fips_stay_the_scrypt_keys_of_the_master_key() -> None: + keys: Final = envelope_keys_from_master_key(_MASTER_KEY) + assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == ( + "34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed", + "eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250", + ) + + +@pytest.mark.usefixtures("openssl_enforcing_fips") +def test_envelope_keys_on_a_fips_openssl_without_fips_mode_stay_the_scrypt_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + keys: Final = envelope_keys_from_master_key(_MASTER_KEY) + assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == ( + "34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed", + "eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250", + ) + + +@pytest.mark.usefixtures("fips_mode_on") +@pytest.mark.parametrize("salt_key", ["", "sk-salt-change-me", _MASTER_KEY, "sk-" + "5a" * 31]) +def test_envelope_keys_under_fips_refuse_a_salt_key_that_is_not_32_bytes_of_hex( + monkeypatch: pytest.MonkeyPatch, salt_key: str +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", salt_key) + with pytest.raises(ValueError, match="LITELLM_SALT_KEY"): + envelope_keys_from_master_key(_MASTER_KEY) diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py index f084d5c7d1d..795ded8528c 100644 --- a/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py @@ -1,6 +1,8 @@ """Tests for the session-token KDF and the edge/token-endpoint resolvers.""" +import _hashlib from datetime import datetime, timedelta, timezone +from typing import Final import pytest from cryptography.hazmat.primitives import serialization @@ -230,3 +232,64 @@ def test_signing_config_error_detail_never_leaks_key_material(): ) assert isinstance(resolved, SessionSigningConfigError) assert pem.splitlines()[1] not in resolved.detail + + +def test_session_key_without_fips_stays_the_scrypt_key_of_the_master_key() -> None: + keys: Final = session_keys_from_master_key(MASTER_KEY) + assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d" + + +@pytest.mark.usefixtures("openssl_enforcing_fips") +def test_session_key_on_a_fips_openssl_without_fips_mode_stays_the_scrypt_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + keys: Final = session_keys_from_master_key(MASTER_KEY) + assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d" + + +def _fips_keys(master_key: str) -> tuple[str, str, str]: + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + envelope: Final = envelope_keys_from_master_key(master_key) + session: Final = session_keys_from_master_key(master_key) + return ( + envelope.signing_key.get_secret_value(), + envelope.encryption_key.get_secret_value(), + session.signing_key.get_secret_value(), + ) + + +_needs_shared_openssl: Final = pytest.mark.skipif( + not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach" +) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_mode_on") +def test_keys_under_fips_come_from_the_salt_key_are_distinct_and_the_same_on_every_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32) + first: Final = _fips_keys(MASTER_KEY) + assert first == _fips_keys(MASTER_KEY) + assert len(set(first)) == 3, first + assert first == ( + "ef694d415772aef09731c47709d4d9da47f7bbfcd6484836dea09c2bb387063b", + "d8a7cd70690fbf98c5581301b2e0460f2ad714f7c74b2af1b3dac3e0aa99db85", + "5e11e470bf9c838165b70d0b2b82a7ee5e2f00d9449eeb54f7c5bf9491b64354", + ) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_mode_on") +def test_keys_under_fips_all_change_when_the_master_key_rotates(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32) + before: Final = _fips_keys(MASTER_KEY) + after: Final = _fips_keys("sk-a-different-master-key") + assert set(before).isdisjoint(after), (before, after) + + +@pytest.mark.usefixtures("fips_mode_on") +def test_session_key_under_fips_refuses_an_unset_salt_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + with pytest.raises(ValueError, match="LITELLM_SALT_KEY"): + session_keys_from_master_key(MASTER_KEY)