From ffbbb6366b8429d0e851cee107f4bde3d333bc03 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 3 Nov 2026 14:56:14 -0800 Subject: [PATCH] feat(proxy): add general_settings.fips_mode startup gate (#48600) * feat(proxy): add LITELLM_FIPS_MODE startup gate With LITELLM_FIPS_MODE=true the proxy refuses to start when TLS verification is off, when Python's OpenSSL does not enforce FIPS, or when the startup password migration fails. A value that is not true/false also refuses. * feat(proxy): refuse FIPS mode when cryptography's OpenSSL is not FIPS The stock cryptography wheel bundles its own non-FIPS OpenSSL, which serves JWT, Fernet and AES-GCM even when Python's OpenSSL enforces FIPS. * fix(proxy): refuse FIPS mode when cryptography's FIPS provider cannot hash The stock cryptography wheel reads the host's FIPS config and reports FIPS on, but its bundled OpenSSL cannot load the host's FIPS provider, so every operation fails. Require a working SHA-256 alongside the FIPS flag. * fix(proxy): refuse TLS verification off anywhere in FIPS mode A model's own ssl_verify: false reached the HTTP client after the startup check passed. get_ssl_verify now raises in FIPS mode whenever verification resolves to off, which also covers the startup check. * refactor(proxy): return FIPS boot refusals as values and harden the FIPS tests * fix(proxy): stop refusing TLS verification off in FIPS mode Skipping certificate verification is a security choice, not a FIPS one: the TLS connection still only uses algorithms the FIPS provider allows. * feat(fips): refuse a database used without FIPS mode, drop the password migration change FIPS mode marks a new database with a fips_mode row in LiteLLM_Config and refuses one that already holds LiteLLM rows without it. Without FIPS mode, the proxy refuses a database carrying the marker. * test(fips): close the database test clients with aclose so the engine stops * feat(fips): read FIPS mode from general_settings.fips_mode instead of LITELLM_FIPS_MODE * refactor(fips): keep fips_mode out of the API-writable general settings, it is read from the config file only * fix(fips): retry the startup marker and state reads once after a dropped connection A dropped connection during the non-FIPS marker read used to count as an unmarked database, so a blip at boot let a non-FIPS proxy start on a FIPS database. Both reads now go through call_with_db_reconnect_retry, the reconnect-and-retry the other startup reads use. A read that still fails keeps its old outcome. * fix(fips): refuse a fips_mode key that has no value fips_mode: os.environ/X with X unset, or a bare fips_mode: / null, resolved to None and started without FIPS mode and without a message, while an empty string refused. A key that is present now has to be a boolean; only a config without the key means off. * refactor(fips): move the database check out of the boot gate The boot gate keeps the setting and the two OpenSSL checks. The check that a database matches the FIPS setting moves to its own PR, so a boot without the flag runs no extra database read. * test(fips): pin how a fips_mode string is read through enforce_fips_mode --- litellm/proxy/common_utils/fips.py | 90 +++++++++++++ litellm/proxy/proxy_server.py | 3 + .../configuration/test_fips_mode_boot.py | 39 ++++++ tests/unit/proxy/common_utils/test_fips.py | 120 ++++++++++++++++++ ...test_proxy_server_endpoints_and_startup.py | 32 +++++ 5 files changed, 284 insertions(+) create mode 100644 litellm/proxy/common_utils/fips.py create mode 100644 tests/integration/configuration/test_fips_mode_boot.py create mode 100644 tests/unit/proxy/common_utils/test_fips.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..37142364af9 --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,90 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.secret_managers.main import str_to_bool + +FIPS_MODE_SETTING: Final = "fips_mode" +_SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}" + + +class FipsModeError(Exception): + pass + + +def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: + """general_settings.fips_mode as a boolean: False when the key is absent, None when it is not a boolean, including + a key with no value.""" + if FIPS_MODE_SETTING not in general_settings: + return False + value: Final = general_settings[FIPS_MODE_SETTING] + if isinstance(value, bool): + return value + return str_to_bool(value) if isinstance(value, str) else None + + +def openssl_enforces_fips() -> bool: + """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" + try: + import _hashlib + + if _hashlib.get_fips_mode() != 1: + return False + except (ImportError, AttributeError, ValueError): + return False + try: + _hashlib.new("md5", b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def cryptography_enforces_fips() -> bool: + """The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256.""" + try: + from cryptography.exceptions import InternalError, UnsupportedAlgorithm + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + from cryptography.hazmat.primitives import hashes + except ImportError: + return False + try: + if not rust_openssl.is_fips_enabled(): + return False + hashes.Hash(hashes.SHA256()).finalize() + except (AttributeError, InternalError, UnsupportedAlgorithm): + return False + return True + + +def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None: + """Why the proxy must not start: general_settings.fips_mode is not a boolean, or is on but Python's or the + cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" + setting: Final = _fips_mode_setting(general_settings) + if setting is None: + value: Final = general_settings.get(FIPS_MODE_SETTING) + if value is None: + return ( + f"{_SETTING} has no value: it is empty, or an os.environ/ reference to a variable that is not set. " + "Set it to true or false, or remove it." + ) + return f"{_SETTING}={value!r} is not a boolean. Set it to true or false, or remove it." + if not setting: + return None + if not openssl_enforces_fips(): + return ( + f"{_SETTING} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with its FIPS " + f"provider active and default_properties = fips=yes, or turn {_SETTING} off." + ) + if not cryptography_enforces_fips(): + return ( + f"{_SETTING} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " + "against the system OpenSSL (pip install --no-binary cryptography cryptography), or turn " + f"{_SETTING} off." + ) + return None + + +def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: + """Raise FipsModeError with the fips_mode_refusal reason, if there is one.""" + refusal: Final = fips_mode_refusal(general_settings) + if refusal is not None: + raise FipsModeError(refusal) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index eedb49ef83f..698c1e6e752 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -530,6 +530,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import enforce_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1673,6 +1674,8 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener if decoded_worker_config is not None: await initialize_from_worker_config(decoded_worker_config) + enforce_fips_mode(general_settings_view()) + async with AsyncExitStack() as boot_database: enforce_master_key_boot_verdict( await with_stored_secrets_counted( diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..18fb692da6e --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,39 @@ +import _hashlib +import json +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway +from tests.integration._support.process import refused_boot_log + + +def _config(directory: Path, upstream_url: str, fips_mode: object) -> Path: + config: Final = directory / "fips_boot.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "fips-model", + "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, + } + ], + "general_settings": {"fips_mode": fips_mode}, + } + ) + ) + return config + + +def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, True)) + assert "general_settings.fips_mode is on but OpenSSL does not enforce FIPS" in log, log + + +def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, "enforced")) + assert "general_settings.fips_mode='enforced' is not a boolean" in log, log diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..a72772df492 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -0,0 +1,120 @@ +import _hashlib +from collections.abc import Mapping +from contextlib import nullcontext +from typing import Final +from unittest.mock import Mock + +import pytest +from cryptography.exceptions import InternalError +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes + +from litellm.proxy.common_utils.fips import ( + FipsModeError, + cryptography_enforces_fips, + enforce_fips_mode, + openssl_enforces_fips, +) + + +def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new: Final = _hashlib.new + + def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: + if name == "md5" and md5_refused: + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, usedforsecurity=usedforsecurity) + + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) + monkeypatch.setattr(_hashlib, "new", new) + + +@pytest.mark.parametrize( + "fips_flag, md5_refused, enforces", + [(1, True, True), (0, True, False), (1, False, False)], + ids=["fips", "fips-flag-off", "md5-allowed"], +) +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5( + monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool +) -> None: + _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) + assert openssl_enforces_fips() is enforces + + +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delattr(_hashlib, "get_fips_mode") + assert openssl_enforces_fips() is False + + +@pytest.mark.parametrize( + "fips_enabled, sha256_broken, enforces", + [(True, False, True), (False, False, False), (True, True, False)], + ids=["fips", "fips-flag-off", "sha256-broken"], +) +def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( + monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool +) -> None: + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) + if sha256_broken: + monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) + assert cryptography_enforces_fips() is enforces + + +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delattr(rust_openssl, "is_fips_enabled") + assert cryptography_enforces_fips() is False + + +def test_this_runners_python_openssl_does_not_enforce_fips() -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's Python OpenSSL is in FIPS mode") + assert openssl_enforces_fips() is False + + +def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: + if rust_openssl.is_fips_enabled(): + pytest.skip("this runner's cryptography OpenSSL is in FIPS mode") + assert cryptography_enforces_fips() is False + + +@pytest.mark.parametrize( + "general_settings, openssl_fips, cryptography_fips, refusal", + [ + ({}, False, False, None), + ({"fips_mode": False}, False, False, None), + ({"fips_mode": "false"}, False, False, None), + ({"fips_mode": True}, True, True, None), + ({"fips_mode": "true"}, True, True, None), + ({"fips_mode": True}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"), + ({"fips_mode": " TRUE "}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"), + ({"fips_mode": True}, True, False, "cryptography library's OpenSSL has no working FIPS provider"), + ({"fips_mode": "enforced"}, True, True, "general_settings.fips_mode='enforced' is not a boolean"), + ({"fips_mode": 1}, True, True, "general_settings.fips_mode=1 is not a boolean"), + ({"fips_mode": None}, True, True, "general_settings.fips_mode has no value"), + ], + ids=[ + "unset", + "false", + "false-from-a-string", + "fips", + "fips-from-a-string", + "openssl-not-fips", + "openssl-not-fips-from-a-padded-string", + "cryptography-not-fips", + "not-a-boolean", + "a-number", + "no-value", + ], +) +def test_enforce_fips_mode( + monkeypatch: pytest.MonkeyPatch, + general_settings: Mapping[str, object], + openssl_fips: bool, + cryptography_fips: bool, + refusal: str | None, +) -> None: + _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + enforce_fips_mode(general_settings) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 9914247bc47..9dd15649d3e 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1794,6 +1794,38 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_reads_fips_mode_from_the_config_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key", "fips_mode": True}) + monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: False) + + with pytest.raises(FipsModeError, match=r"general_settings\.fips_mode is on but OpenSSL does not enforce FIPS"): + async with proxy_startup_event(FastAPI()): + pass + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_a_fips_mode_reference_to_an_unset_variable( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings( + monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key", "fips_mode": "os.environ/FIPS_MODE_NOT_SET"} + ) + monkeypatch.delenv("FIPS_MODE_NOT_SET", raising=False) + + with pytest.raises(FipsModeError, match=r"general_settings\.fips_mode has no value"): + async with proxy_startup_event(FastAPI()): + pass + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext