From 6103a8c60c16b2641d73eacd279ced26b6bf2f7e Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 18:33:09 -0700 Subject: [PATCH] refactor(proxy): return FIPS boot refusals as values and harden the FIPS tests --- litellm/llms/custom_httpx/http_handler.py | 25 ++++++---- litellm/proxy/common_utils/fips.py | 31 ++++++++---- .../configuration/test_fips_mode_boot.py | 4 +- tests/unit/proxy/common_utils/test_fips.py | 46 ++++++++++++------ ...test_proxy_server_endpoints_and_startup.py | 48 +++++++++++++++---- 5 files changed, 109 insertions(+), 45 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index ae9aa8f19d2..81cd624883b 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -336,6 +336,21 @@ def _create_ssl_context( def get_ssl_verify( ssl_verify: bool | str | None = None, +) -> bool | str: + """resolve_ssl_verify, refusing disabled TLS verification under LITELLM_FIPS_MODE.""" + resolved: Final = resolve_ssl_verify(ssl_verify) + if resolved is not False: + return resolved + + from litellm.proxy.common_utils.fips import TLS_VERIFICATION_DISABLED, FipsModeError, is_fips_mode + + if is_fips_mode(): + raise FipsModeError(TLS_VERIFICATION_DISABLED) + return resolved + + +def resolve_ssl_verify( + ssl_verify: bool | str | None = None, ) -> bool | str: """ Common utility to resolve the SSL verification setting. @@ -370,16 +385,6 @@ def get_ssl_verify( if ssl_cert_file and os.path.exists(ssl_cert_file): return ssl_cert_file - if ssl_verify is False: - from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode - - if is_fips_mode(): - raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " - "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a " - "CA bundle." - ) - return ssl_verify if ssl_verify is not None else True diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 2ffb17852eb..2cbe154d2a1 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -4,6 +4,10 @@ from typing import Final from litellm.secret_managers.main import str_to_bool FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +TLS_VERIFICATION_DISABLED: Final = ( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " + "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a CA bundle." +) class FipsModeError(Exception): @@ -47,26 +51,35 @@ def cryptography_enforces_fips() -> bool: return True -def enforce_fips_mode() -> None: - """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or Python's or the cryptography library's OpenSSL does not enforce FIPS.""" +def fips_mode_refusal() -> str | None: + """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off + or Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: - raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") + return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it." if not is_fips_mode(): - return + return None - from litellm.llms.custom_httpx.http_handler import get_ssl_verify + from litellm.llms.custom_httpx.http_handler import resolve_ssl_verify - get_ssl_verify() # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off + if resolve_ssl_verify() is False: + return TLS_VERIFICATION_DISABLED if not openssl_enforces_fips(): - raise FipsModeError( + return ( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." ) if not cryptography_enforces_fips(): - raise FipsModeError( + return ( f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " f"{FIPS_MODE_ENV_VAR}." ) + return None + + +def enforce_fips_mode() -> None: + """Raise FipsModeError with the fips_mode_refusal reason, if there is one.""" + refusal: Final = fips_mode_refusal() + if refusal is not None: + raise FipsModeError(refusal) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index 85c7011876a..0a20d14fb19 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -58,5 +58,7 @@ def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Ga def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) - with owned_proxy(gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config) as proxy: + with owned_proxy( + gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config, remove_environment=("LITELLM_FIPS_MODE",) + ) as proxy: assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index e70fd2e3b46..596d45f88d1 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,5 +1,6 @@ import _hashlib from contextlib import nullcontext +from typing import Final from unittest.mock import Mock import pytest @@ -18,20 +19,20 @@ from litellm.proxy.common_utils.fips import ( ) -def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: - real_new = _hashlib.new +def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new: Final = _hashlib.new - def new(name, data=b"", **kwargs): + def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: if name == "md5" and md5_refused: raise _hashlib.UnsupportedDigestmodError("unsupported hash type") - return real_new(name, data, **kwargs) + return real_new(name, data, usedforsecurity=usedforsecurity) monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) monkeypatch.setattr(_hashlib, "new", new) @pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) -def test_is_fips_mode(monkeypatch, value, expected): +def test_is_fips_mode(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool) -> None: if value is None: monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) else: @@ -44,12 +45,14 @@ def test_is_fips_mode(monkeypatch, value, expected): [(1, True, True), (0, True, False), (1, False, False)], ids=["fips", "fips-flag-off", "md5-allowed"], ) -def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(monkeypatch, fips_flag, md5_refused, enforces): +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5( + monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool +) -> None: _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) assert openssl_enforces_fips() is enforces -def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(_hashlib, "get_fips_mode") assert openssl_enforces_fips() is False @@ -60,23 +63,28 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): ids=["fips", "fips-flag-off", "sha256-broken"], ) def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( - monkeypatch, fips_enabled, sha256_broken, enforces -): + monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool +) -> None: monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) if sha256_broken: monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) assert cryptography_enforces_fips() is enforces -def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(rust_openssl, "is_fips_enabled") assert cryptography_enforces_fips() is False -def test_this_runners_openssl_does_not_enforce_fips(): +def test_this_runners_python_openssl_does_not_enforce_fips() -> None: if _hashlib.get_fips_mode() == 1: - pytest.skip("this runner's OpenSSL is in FIPS mode") + pytest.skip("this runner's Python OpenSSL is in FIPS mode") assert openssl_enforces_fips() is False + + +def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: + if rust_openssl.is_fips_enabled(): + pytest.skip("this runner's cryptography OpenSSL is in FIPS mode") assert cryptography_enforces_fips() is False @@ -106,8 +114,14 @@ def test_this_runners_openssl_does_not_enforce_fips(): ], ) def test_enforce_fips_mode( - monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal -): + monkeypatch: pytest.MonkeyPatch, + fips_mode: str | None, + ssl_verify_env: str | None, + ssl_verify_setting: bool, + openssl_fips: bool, + cryptography_fips: bool, + refusal: str | None, +) -> None: for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): if value is None: monkeypatch.delenv(name, raising=False) @@ -122,7 +136,9 @@ def test_enforce_fips_mode( @pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) -def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused): +def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, refused: bool +) -> None: monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): assert get_ssl_verify(ssl_verify=False) is False diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 031c772dc44..4bdadbba02e 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1776,7 +1776,9 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.proxy_server import proxy_startup_event @@ -1791,22 +1793,48 @@ async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_veri pass +class _UsersWithOnePlaintextPassword: + async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: + return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) + + async def update(self, where: object, data: object) -> None: + raise AssertionError("a refused boot must not write to the database") + + +class _DatabaseWithOnePlaintextPassword: + def __init__(self) -> None: + self.db: Final = types.SimpleNamespace(litellm_usertable=_UsersWithOnePlaintextPassword()) + + @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(monkeypatch, tmp_path): +async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import _hashlib + import hashlib + + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + from litellm.proxy.common_utils.fips import FipsModeError - from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_startup_event + from litellm.proxy.proxy_server import proxy_startup_event + + real_new: Final = _hashlib.new + + def new_refusing_md5(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: + if name == "md5": + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, usedforsecurity=usedforsecurity) _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) monkeypatch.setattr(litellm, "ssl_verify", True) monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) - monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) - monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) - scrypt_refused = ValueError("[digital envelope routines] unsupported") - monkeypatch.setattr( - "litellm.proxy.proxy_server.migrate_passwords_to_scrypt_async", AsyncMock(side_effect=scrypt_refused) - ) + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr(_hashlib, "new", new_refusing_md5) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + scrypt_refused: Final = ValueError("[digital envelope routines] unsupported") + monkeypatch.setattr(hashlib, "scrypt", mock.Mock(side_effect=scrypt_refused)) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _DatabaseWithOnePlaintextPassword()) with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: async with proxy_startup_event(FastAPI()):