From eba880e108109fec9a9a131c6e26c71b9c2b5828 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 11 Nov 2026 00:48:11 -0800 Subject: [PATCH] fix(tls)!: exclude RSA key exchange and ChaCha20 from the outbound cipher list in FIPS mode (#50671) * fix(tls): exclude RSA key exchange from outbound cipher list in FIPS mode * fix(tls): apply the FIPS cipher filter to SSL contexts clients already hold enforce_fips_mode runs after load_config, so guardrail and logger clients built during config load held an SSL context without the RSA key exchange exclusion. enforce_fips_mode now updates every cached context in place, and the SDK no longer imports the proxy's FIPS module. * fix(tls): refuse to start in FIPS mode when SSL_SECURITY_LEVEL has only RSA key exchange Excluding RSA key exchange from an SSL_SECURITY_LEVEL that lists only RSA key-exchange suites leaves no TLS 1.2 suite, so set_ciphers raised on every client build. enforce_fips_mode now checks the configured list first and refuses to start with a FipsModeError that names the setting. The cipher list test no longer assumes the host has RSA key-exchange suites. * fix(tls): offer only TLS 1.3 instead of refusing to start when no TLS 1.2 suite without RSA key exchange is left * chore(tls): warn when fips_mode leaves a cipher list with only TLS 1.3 * style(tls): split the TLS 1.3 fallback warning to fit the line limit * fix(proxy): exclude RSA key exchange before config-load secret reads in FIPS mode get_config reads secrets (AWS, GCP, HashiCorp, CyberArk secret managers) over the shared outbound SSL context before enforce_fips_mode runs, so with general_settings.fips_mode true those first TLS handshakes still offered RSA key exchange. Turn the exclusion on in get_config from the raw general_settings, right before the secret manager starts, reusing the existing fips_mode_setting (now public) and exclude_rsa_key_exchange. enforce_fips_mode still owns the refusal and the final state; fips_mode given as an os.environ/ reference is still handled there. * test(proxy): skip the config-load RSA key exchange test on an OpenSSL without RSA suites On an OpenSSL whose default cipher list offers no RSA key exchange suite, fips and standard mode read secrets the same way, so the test has nothing to tell apart and skips, as the http_handler fallback test already does. * fix(tls): exclude TLS 1.2 ChaCha20 suites from the outbound cipher list in FIPS mode ChaCha20-Poly1305 is not a FIPS approved algorithm. FIPS providers already refuse it, so on the Chainguard image this changes nothing, but fips_mode now drops it from TLS 1.2 explicitly next to RSA key exchange. Python's set_ciphers cannot touch TLS 1.3 suites, which still come from OpenSSL's config. exclude_rsa_key_exchange is renamed to exclude_rsa_key_exchange_and_chacha20 to match. --- litellm/llms/custom_httpx/http_handler.py | 55 +++++++-- litellm/proxy/common_utils/fips.py | 9 +- litellm/proxy/proxy_server.py | 9 +- .../llms/custom_httpx/test_http_handler.py | 108 ++++++++++++++++++ .../proxy/proxy_server/test_proxy_config.py | 70 ++++++++++++ 5 files changed, 234 insertions(+), 17 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 307d67bab21..011551faf0c 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -316,6 +316,48 @@ def _prepare_request_data_and_content( # Cache for SSL contexts to avoid creating duplicate contexts with the same configuration _ssl_context_cache: Final[dict[tuple[str | None, str | None, str | None, bool], ssl.SSLContext]] = {} +_rsa_key_exchange_and_chacha20_excluded: Final = threading.Event() + + +def _set_ciphers( + context: ssl.SSLContext, ssl_security_level: str | None, rsa_key_exchange_and_chacha20_excluded: bool +) -> None: + """ + Offer TLS 1.2 and up with SSL_SECURITY_LEVEL's cipher string if set, else DEFAULT_SSL_CIPHERS. + With RSA key exchange and ChaCha20 excluded, drop their TLS 1.2 suites, or offer only TLS 1.3 and log a warning + when the string has no other TLS 1.2 suite. TLS 1.3 suites come from OpenSSL's config, which set_ciphers can't + change. + """ + ciphers: Final = ( + ssl_security_level if ssl_security_level and isinstance(ssl_security_level, str) else DEFAULT_SSL_CIPHERS + ) + context.minimum_version = ssl.TLSVersion.TLSv1_2 # rebind-ok: sets up the caller's context in place + context.set_ciphers(ciphers) + if rsa_key_exchange_and_chacha20_excluded: + try: + context.set_ciphers(f"{ciphers}:!kRSA:!CHACHA20") + except ssl.SSLError: + context.minimum_version = ssl.TLSVersion.TLSv1_3 # rebind-ok: sets up the caller's context in place + verbose_logger.warning( + "fips_mode: cipher list '%s' has no TLS 1.2 suite without RSA key exchange or ChaCha20, " + "so outbound TLS offers only TLS 1.3 and TLS 1.2-only servers will fail.", + ciphers, + ) + + +def exclude_rsa_key_exchange_and_chacha20(excluded: bool) -> None: + """ + Turn RSA key exchange and TLS 1.2 ChaCha20 off or back on for new SSL contexts and every cached one, which clients + already hold. + """ + if excluded == _rsa_key_exchange_and_chacha20_excluded.is_set(): + return + if excluded: + _rsa_key_exchange_and_chacha20_excluded.set() + else: + _rsa_key_exchange_and_chacha20_excluded.clear() + for (_, ssl_security_level, _, _), context in list(_ssl_context_cache.items()): + _set_ciphers(context, ssl_security_level, excluded) def _create_ssl_context( @@ -329,18 +371,7 @@ def _create_ssl_context( """ custom_ssl_context: Final = ssl.create_default_context(cafile=cafile) - # Optimize SSL handshake performance - # Set minimum TLS version to 1.2 for better performance - custom_ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2 - - # Configure cipher suites for optimal performance - if ssl_security_level and isinstance(ssl_security_level, str): - # User provided custom cipher configuration (e.g., via SSL_SECURITY_LEVEL env var) - custom_ssl_context.set_ciphers(ssl_security_level) - else: - # Use optimized cipher list that strongly prefers fast ciphers - # but falls back to widely compatible ones - custom_ssl_context.set_ciphers(DEFAULT_SSL_CIPHERS) + _set_ciphers(custom_ssl_context, ssl_security_level, _rsa_key_exchange_and_chacha20_excluded.is_set()) # Configure ECDH curve for key exchange (e.g., to disable PQC and improve performance) # Set SSL_ECDH_CURVE env var or litellm.ssl_ecdh_curve to 'X25519' to disable PQC diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 6cd23e5c0ff..efa57817482 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -2,6 +2,7 @@ import threading from collections.abc import Mapping from typing import Final +from litellm.llms.custom_httpx.http_handler import exclude_rsa_key_exchange_and_chacha20 from litellm.secret_managers.main import str_to_bool FIPS_MODE_SETTING: Final = "fips_mode" @@ -13,7 +14,7 @@ class FipsModeError(Exception): pass -def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: +def fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: """general_settings.fips_mode as a boolean: False when the key is absent, None when it is not a boolean, including a key with no value.""" if FIPS_MODE_SETTING not in general_settings: @@ -60,7 +61,7 @@ def cryptography_enforces_fips() -> bool: def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None: """Why the proxy must not start: general_settings.fips_mode is not a boolean, or is on but Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" - setting: Final = _fips_mode_setting(general_settings) + setting: Final = fips_mode_setting(general_settings) if setting is None: value: Final = general_settings.get(FIPS_MODE_SETTING) if value is None: @@ -89,7 +90,9 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: refusal: Final = fips_mode_refusal(general_settings) if refusal is not None: raise FipsModeError(refusal) - if _fips_mode_setting(general_settings): + active: Final = fips_mode_setting(general_settings) is True + exclude_rsa_key_exchange_and_chacha20(active) + if active: _FIPS_MODE_ACTIVE.set() else: _FIPS_MODE_ACTIVE.clear() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index de98a4d571b..9f721e3fdcb 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -419,7 +419,7 @@ from litellm.llms.anthropic.messages.convertible_blocks import ( malformed_tool_result_count_error, ) from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, exclude_rsa_key_exchange_and_chacha20 from litellm.llms.openai.live import parse_live_session_start from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -547,7 +547,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id -from litellm.proxy.common_utils.fips import enforce_fips_mode +from litellm.proxy.common_utils.fips import enforce_fips_mode, fips_mode_setting from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -6680,6 +6680,11 @@ class ProxyConfig: printed_yaml: Final = copy.deepcopy(config) printed_yaml.pop("environment_variables", None) + # Secret manager reads and the rest of config load make outbound TLS calls before enforce_fips_mode runs + raw_general_settings: Final = as_str_mapping(config.get("general_settings")) + if raw_general_settings is not None and fips_mode_setting(raw_general_settings) is True: + exclude_rsa_key_exchange_and_chacha20(True) + self._initialize_secret_manager_from_raw_config(config=config, config_file_path=config_file_path) config = self._check_for_os_environ_vars(config=config) diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index 4e031531180..adb6410d360 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -339,6 +339,114 @@ def test_get_ssl_configuration_integration(): assert ssl_context.verify_mode is not None +def _offered_ciphers(context: bool | str | ssl.SSLContext) -> tuple[tuple[str, str, str, str | None], ...]: + assert isinstance(context, ssl.SSLContext) + return tuple( + (cipher["name"], cipher["protocol"], cipher["kea"], cipher["symmetric"]) for cipher in context.get_ciphers() + ) + + +def _kept_in_fips_mode(cipher: tuple[str, str, str, str | None]) -> bool: + _, protocol, kea, symmetric = cipher + return kea != "kx-rsa" and (protocol, symmetric) != ("TLSv1.2", "chacha20-poly1305") + + +def _use_ssl_security_level(monkeypatch: pytest.MonkeyPatch, ssl_security_level: str | None) -> None: + from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + + _ssl_context_cache.clear() + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE", "SSL_CERT_FILE", "SSL_VERIFY"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "ssl_security_level", ssl_security_level) + + +def _fake_fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None: + import _hashlib + + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr( + _hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported")) + ) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + + +@pytest.mark.parametrize( + "ssl_security_level", + [None, "AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-GCM-SHA384"], + ids=["default-cipher-list", "ssl-security-level"], +) +def test_fips_mode_drops_rsa_key_exchange_and_tls_1_2_chacha20_from_the_outbound_cipher_list( + monkeypatch: pytest.MonkeyPatch, ssl_security_level: str | None +) -> None: + from litellm.constants import DEFAULT_SSL_CIPHERS + from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + from litellm.proxy.common_utils.fips import enforce_fips_mode + + _use_ssl_security_level(monkeypatch, ssl_security_level) + configured: Final = ssl.create_default_context() + configured.set_ciphers(ssl_security_level or DEFAULT_SSL_CIPHERS) + + # A client built while the config loads holds its context from before FIPS mode is enforced + held: Final = get_ssl_configuration() + standard: Final = _offered_ciphers(held) + _fake_fips_openssl(monkeypatch) + enforce_fips_mode({"fips_mode": True}) + try: + held_in_fips: Final = _offered_ciphers(held) + new_in_fips: Final = _offered_ciphers(get_ssl_configuration(http2=True)) + finally: + enforce_fips_mode({}) + held_after_fips: Final = _offered_ciphers(held) + _ssl_context_cache.clear() + + assert standard == _offered_ciphers(configured) + kept_in_fips: Final = tuple(filter(_kept_in_fips_mode, standard)) + assert held_in_fips == kept_in_fips + assert new_in_fips == kept_in_fips + assert held_after_fips == standard + + +@pytest.mark.parametrize("excluded_suites", ["kRSA", "CHACHA20"]) +def test_fips_mode_offers_only_tls_1_3_when_the_cipher_list_has_only_excluded_tls_1_2_suites( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + excluded_suites: str, +) -> None: + from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + from litellm.proxy.common_utils.fips import enforce_fips_mode, fips_mode_active + + excluded: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + try: + excluded.set_ciphers(excluded_suites) + except ssl.SSLError: + pytest.skip(f"this OpenSSL has no {excluded_suites} suites") + _use_ssl_security_level( + monkeypatch, + ":".join(cipher["name"] for cipher in excluded.get_ciphers() if cipher["protocol"] != "TLSv1.3"), + ) + held: Final = get_ssl_configuration() + assert isinstance(held, ssl.SSLContext) + _fake_fips_openssl(monkeypatch) + caplog.set_level(logging.WARNING, logger="LiteLLM") + enforce_fips_mode({"fips_mode": True}) + try: + started_in_fips: Final = fips_mode_active() + new: Final = get_ssl_configuration(http2=True) + assert isinstance(new, ssl.SSLContext) + in_fips: Final = (held.minimum_version, new.minimum_version) + finally: + enforce_fips_mode({}) + after_fips: Final = held.minimum_version + _ssl_context_cache.clear() + + assert started_in_fips is True + assert in_fips == (ssl.TLSVersion.TLSv1_3, ssl.TLSVersion.TLSv1_3) + assert after_fips == ssl.TLSVersion.TLSv1_2 + assert any("offers only TLS 1.3" in record.getMessage() for record in caplog.records) + + # Session Reuse Tests class MockClientSession: """Mock ClientSession that is not callable""" diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 712b4134c05..91ff818da4c 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -14,6 +14,7 @@ import json import logging import os import re +import ssl import sys from collections.abc import Awaitable, Callable, Mapping from contextlib import nullcontext @@ -34,9 +35,11 @@ from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import turn_on_json +from litellm.constants import DEFAULT_SSL_CIPHERS from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.common_utils.fips import enforce_fips_mode from litellm.proxy.config_resolvers.settings_store import SettingsStore from litellm.router_api import ApplyResult, RouterSettings from litellm.types.router import Deployment, LiteLLM_Params @@ -1901,6 +1904,73 @@ async def test_ProxyConfig_get_config_reuses_an_already_initialized_secret_manag } == {"client_reused": True, "master_key": "master-from-vault"} +TLS_RECORDING_SECRET_MANAGER_MODULE = """ +import os + +from litellm.integrations.custom_secret_manager import CustomSecretManager +from litellm.llms.custom_httpx.http_handler import get_ssl_configuration + + +class TlsRecordingSecretManager(CustomSecretManager): + def sync_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + offers_rsa = any(c["kea"] == "kx-rsa" for c in get_ssl_configuration().get_ciphers()) + with open(os.environ["RSA_OFFER_LOG"], "a") as f: + f.write(f"{secret_name} {offers_rsa}\\n") + return "provider-from-vault" + + async def async_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): + return self.sync_read_secret(secret_name) +""" + +TLS_RECORDING_CONFIG = """ +model_list: + - model_name: my-model + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/MY_PROVIDER_KEY + +general_settings: + fips_mode: {fips_mode} + key_management_system: custom + key_management_settings: + custom_secret_manager: tls_recording_secret_manager.TlsRecordingSecretManager + hosted_keys: + - MY_PROVIDER_KEY +""" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, rsa_offered", + [("true", False), ('"true"', False), ("false", True)], + ids=["fips-mode", "fips-mode-from-a-string", "standard-mode"], +) +async def test_ProxyConfig_get_config_reads_secrets_without_rsa_key_exchange_in_fips_mode( + tmp_path, monkeypatch, fips_mode: str, rsa_offered: bool +): + """Secret reads during get_config run before enforce_fips_mode, so fips_mode already excludes RSA key exchange.""" + default_cipher_list: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + default_cipher_list.set_ciphers(DEFAULT_SSL_CIPHERS) + if not any(cipher["kea"] == "kx-rsa" for cipher in default_cipher_list.get_ciphers()): + pytest.skip("this OpenSSL offers no RSA key exchange suite from DEFAULT_SSL_CIPHERS") + config_file_path = _write_vault_backed_config( + tmp_path, monkeypatch, TLS_RECORDING_CONFIG.format(fips_mode=fips_mode) + ) + (tmp_path / "tls_recording_secret_manager.py").write_text(TLS_RECORDING_SECRET_MANAGER_MODULE) + rsa_offer_log: Final = tmp_path / "rsa_offers.log" + monkeypatch.setenv("RSA_OFFER_LOG", str(rsa_offer_log)) + + try: + cfg = await ProxyConfig().get_config(config_file_path=config_file_path) + finally: + enforce_fips_mode({}) + + assert { + "reads": set(rsa_offer_log.read_text().splitlines()), + "api_key": cfg["model_list"][0]["litellm_params"]["api_key"], + } == {"reads": {f"MY_PROVIDER_KEY {rsa_offered}"}, "api_key": "provider-from-vault"} + + @pytest.mark.asyncio async def test_ProxyConfig_get_config_without_key_management_system_leaves_secret_manager_unset(tmp_path, monkeypatch): """No ``key_management_system`` means no manager, an unresolvable reference stays None, and