From db7c9aa5469d0e1e8f09cc959be7c6efa06e6eb8 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 10:31:54 -0700 Subject: [PATCH 1/5] feat(proxy): add LITELLM_FIPS_MODE startup gate With LITELLM_FIPS_MODE=true the proxy refuses to start when TLS verification is off, when Python's OpenSSL does not enforce FIPS, or when the startup password migration fails. A value that is not true/false also refuses. --- litellm/proxy/common_utils/fips.py | 53 ++++++++++++ litellm/proxy/proxy_server.py | 10 ++- .../configuration/test_fips_mode_boot.py | 62 ++++++++++++++ tests/unit/proxy/common_utils/test_fips.py | 85 +++++++++++++++++++ ...test_proxy_server_endpoints_and_startup.py | 39 +++++++++ 5 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/common_utils/fips.py create mode 100644 tests/integration/configuration/test_fips_mode_boot.py create mode 100644 tests/unit/proxy/common_utils/test_fips.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..3a8df10b9e5 --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,53 @@ +import os +from typing import Final + +from litellm.secret_managers.main import str_to_bool + +FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" + + +class FipsModeError(Exception): + pass + + +def is_fips_mode() -> bool: + return str_to_bool(os.getenv(FIPS_MODE_ENV_VAR)) is True + + +def openssl_enforces_fips() -> bool: + """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" + try: + import _hashlib + + if _hashlib.get_fips_mode() != 1: + return False + except (ImportError, AttributeError, ValueError): + return False + try: + _hashlib.new("md5", b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def enforce_fips_mode() -> None: + """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off + or OpenSSL does not enforce FIPS.""" + value: Final = os.getenv(FIPS_MODE_ENV_VAR) + if value and str_to_bool(value) is None: + raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") + if not is_fips_mode(): + return + + from litellm.llms.custom_httpx.http_handler import get_ssl_verify + + if get_ssl_verify() is False: + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY or " + "litellm_settings.ssl_verify. Remove the override, or point ssl_verify at a CA bundle." + ) + if not openssl_enforces_fips(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " + f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bcba76ca1b3..5aacc7ad1e6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -489,6 +489,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1523,6 +1524,8 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState if decoded_worker_config is not None: await initialize_from_worker_config(decoded_worker_config) + enforce_fips_mode() + enforce_master_key_boot_verdict( await with_stored_secrets_counted( master_key_boot_verdict( @@ -1565,9 +1568,14 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState result: Final = await migrate_passwords_to_scrypt_async(prisma_client) verbose_proxy_logger.info("Password migration: %s", result) except Exception as e: + if is_fips_mode(): + raise FipsModeError(f"Password migration failed, so plaintext passwords stay unhashed: {e}") from e verbose_proxy_logger.warning("Password migration skipped: %s", e) - asyncio.create_task(_run_pw_migration()) + if is_fips_mode(): + await _run_pw_migration() + else: + asyncio.create_task(_run_pw_migration()) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..85c7011876a --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,62 @@ +import _hashlib +import json +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway +from tests.integration._support.process import owned_proxy, refused_boot_log + + +def _config(directory: Path, upstream_url: str, *, ssl_verify: bool = True) -> Path: + config: Final = directory / "fips_boot.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "fips-model", + "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, + } + ], + "litellm_settings": {"ssl_verify": ssl_verify}, + } + ) + ) + return config + + +def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + log: Final = refused_boot_log( + gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=_config(tmp_path, gateway.upstream_url) + ) + assert "LITELLM_FIPS_MODE is on but OpenSSL does not enforce FIPS" in log, log + + +@pytest.mark.parametrize( + "overrides, ssl_verify", + [({"SSL_VERIFY": "false"}, True), ({}, False)], + ids=["ssl-verify-env", "litellm-settings"], +) +def test_fips_mode_refuses_to_start_with_tls_verification_disabled( + gateway: Gateway, tmp_path: Path, overrides: dict[str, str], ssl_verify: bool +) -> None: + config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=ssl_verify) + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", **overrides}, config=config) + assert "LITELLM_FIPS_MODE is on but TLS certificate verification is disabled" in log, log + + +def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log( + gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) + ) + assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log + + +def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: + config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) + with owned_proxy(gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config) as proxy: + assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..392db0d9cb4 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -0,0 +1,85 @@ +import _hashlib +from contextlib import nullcontext + +import pytest + +import litellm +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode, openssl_enforces_fips + + +def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new = _hashlib.new + + def new(name, data=b"", **kwargs): + if name == "md5" and md5_refused: + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, **kwargs) + + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) + monkeypatch.setattr(_hashlib, "new", new) + + +@pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) +def test_is_fips_mode(monkeypatch, value, expected): + if value is None: + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + else: + monkeypatch.setenv("LITELLM_FIPS_MODE", value) + assert is_fips_mode() is expected + + +@pytest.mark.parametrize( + "fips_flag, md5_refused, enforces", + [(1, True, True), (0, True, False), (1, False, False)], + ids=["fips", "fips-flag-off", "md5-allowed"], +) +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(monkeypatch, fips_flag, md5_refused, enforces): + _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) + assert openssl_enforces_fips() is enforces + + +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): + monkeypatch.delattr(_hashlib, "get_fips_mode") + assert openssl_enforces_fips() is False + + +def test_this_runners_openssl_does_not_enforce_fips(): + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + assert openssl_enforces_fips() is False + + +@pytest.mark.parametrize( + "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal", + [ + (None, "false", False, False, None), + ("", "false", False, False, None), + ("false", "false", False, False, None), + ("true", None, True, True, None), + ("true", "false", True, True, "TLS certificate verification is disabled"), + ("true", None, False, True, "TLS certificate verification is disabled"), + ("true", None, True, False, "OpenSSL does not enforce FIPS"), + ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + ], + ids=[ + "unset", + "empty", + "false", + "fips", + "ssl-verify-env-off", + "ssl-verify-setting-off", + "openssl-not-fips", + "not-a-boolean", + ], +) +def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal): + for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) + monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) + _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + enforce_fips_mode() diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 745488bf93f..12f0161ab0c 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1775,6 +1775,45 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + config_path, _ = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + config_path.write_text(yaml.dump({"litellm_settings": {"ssl_verify": False}})) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError, match="TLS certificate verification is disabled"): + async with proxy_startup_event(FastAPI()): + pass + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(monkeypatch, tmp_path): + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) + monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) + scrypt_refused = ValueError("[digital envelope routines] unsupported") + monkeypatch.setattr( + "litellm.proxy.proxy_server.migrate_passwords_to_scrypt_async", AsyncMock(side_effect=scrypt_refused) + ) + + with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: + async with proxy_startup_event(FastAPI()): + pass + + assert refused.value.__cause__ is scrypt_refused + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext From 75c6d07a8f87d7beb92e4304be8da24bf5046aa2 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 14:17:56 -0700 Subject: [PATCH 2/5] feat(proxy): refuse FIPS mode when cryptography's OpenSSL is not FIPS The stock cryptography wheel bundles its own non-FIPS OpenSSL, which serves JWT, Fernet and AES-GCM even when Python's OpenSSL enforces FIPS. --- litellm/proxy/common_utils/fips.py | 17 ++++++- tests/unit/proxy/common_utils/test_fips.py | 46 ++++++++++++++----- ...test_proxy_server_endpoints_and_startup.py | 1 + 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 3a8df10b9e5..f57f56d3248 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -30,9 +30,19 @@ def openssl_enforces_fips() -> bool: return False +def cryptography_enforces_fips() -> bool: + """The cryptography library's own OpenSSL runs with FIPS-only defaults.""" + try: + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + + return rust_openssl.is_fips_enabled() + except (ImportError, AttributeError): + return False + + def enforce_fips_mode() -> None: """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or OpenSSL does not enforce FIPS.""" + or Python's or the cryptography library's OpenSSL does not enforce FIPS.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") @@ -51,3 +61,8 @@ def enforce_fips_mode() -> None: f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." ) + if not cryptography_enforces_fips(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL does not enforce FIPS. Build it against " + f"the system OpenSSL (pip install --no-binary cryptography cryptography), or unset {FIPS_MODE_ENV_VAR}." + ) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 392db0d9cb4..98e95263dcb 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -2,9 +2,16 @@ import _hashlib from contextlib import nullcontext import pytest +from cryptography.hazmat.bindings._rust import openssl as rust_openssl import litellm -from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode, openssl_enforces_fips +from litellm.proxy.common_utils.fips import ( + FipsModeError, + cryptography_enforces_fips, + enforce_fips_mode, + is_fips_mode, + openssl_enforces_fips, +) def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: @@ -43,23 +50,36 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): assert openssl_enforces_fips() is False +@pytest.mark.parametrize("fips_enabled", [True, False]) +def test_cryptography_enforces_fips_reads_its_openssl_fips_flag(monkeypatch, fips_enabled): + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) + assert cryptography_enforces_fips() is fips_enabled + + +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): + monkeypatch.delattr(rust_openssl, "is_fips_enabled") + assert cryptography_enforces_fips() is False + + def test_this_runners_openssl_does_not_enforce_fips(): if _hashlib.get_fips_mode() == 1: pytest.skip("this runner's OpenSSL is in FIPS mode") assert openssl_enforces_fips() is False + assert cryptography_enforces_fips() is False @pytest.mark.parametrize( - "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal", + "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal", [ - (None, "false", False, False, None), - ("", "false", False, False, None), - ("false", "false", False, False, None), - ("true", None, True, True, None), - ("true", "false", True, True, "TLS certificate verification is disabled"), - ("true", None, False, True, "TLS certificate verification is disabled"), - ("true", None, True, False, "OpenSSL does not enforce FIPS"), - ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + (None, "false", False, False, False, None), + ("", "false", False, False, False, None), + ("false", "false", False, False, False, None), + ("true", None, True, True, True, None), + ("true", "false", True, True, True, "TLS certificate verification is disabled"), + ("true", None, False, True, True, "TLS certificate verification is disabled"), + ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), + ("true", None, True, True, False, "cryptography library's OpenSSL does not enforce FIPS"), + ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], ids=[ "unset", @@ -69,10 +89,13 @@ def test_this_runners_openssl_does_not_enforce_fips(): "ssl-verify-env-off", "ssl-verify-setting-off", "openssl-not-fips", + "cryptography-not-fips", "not-a-boolean", ], ) -def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal): +def test_enforce_fips_mode( + monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal +): for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): if value is None: monkeypatch.delenv(name, raising=False) @@ -80,6 +103,7 @@ def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_se monkeypatch.setenv(name, value) monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 12f0161ab0c..031c772dc44 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1801,6 +1801,7 @@ async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) + monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) scrypt_refused = ValueError("[digital envelope routines] unsupported") monkeypatch.setattr( From 23ebd158bf15861b102afd86e0edb090b54e450e Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 16:33:22 -0700 Subject: [PATCH 3/5] fix(proxy): refuse FIPS mode when cryptography's FIPS provider cannot hash The stock cryptography wheel reads the host's FIPS config and reports FIPS on, but its bundled OpenSSL cannot load the host's FIPS provider, so every operation fails. Require a working SHA-256 alongside the FIPS flag. --- litellm/proxy/common_utils/fips.py | 20 ++++++++++++++------ tests/unit/proxy/common_utils/test_fips.py | 19 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index f57f56d3248..671c023bbdb 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -31,13 +31,20 @@ def openssl_enforces_fips() -> bool: def cryptography_enforces_fips() -> bool: - """The cryptography library's own OpenSSL runs with FIPS-only defaults.""" + """The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256.""" try: + from cryptography.exceptions import InternalError, UnsupportedAlgorithm from cryptography.hazmat.bindings._rust import openssl as rust_openssl - - return rust_openssl.is_fips_enabled() - except (ImportError, AttributeError): + from cryptography.hazmat.primitives import hashes + except ImportError: return False + try: + if not rust_openssl.is_fips_enabled(): + return False + hashes.Hash(hashes.SHA256()).finalize() + except (AttributeError, InternalError, UnsupportedAlgorithm): + return False + return True def enforce_fips_mode() -> None: @@ -63,6 +70,7 @@ def enforce_fips_mode() -> None: ) if not cryptography_enforces_fips(): raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL does not enforce FIPS. Build it against " - f"the system OpenSSL (pip install --no-binary cryptography cryptography), or unset {FIPS_MODE_ENV_VAR}." + f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " + "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " + f"{FIPS_MODE_ENV_VAR}." ) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 98e95263dcb..bd6834af536 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,8 +1,11 @@ import _hashlib from contextlib import nullcontext +from unittest.mock import Mock import pytest +from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes import litellm from litellm.proxy.common_utils.fips import ( @@ -50,10 +53,18 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): assert openssl_enforces_fips() is False -@pytest.mark.parametrize("fips_enabled", [True, False]) -def test_cryptography_enforces_fips_reads_its_openssl_fips_flag(monkeypatch, fips_enabled): +@pytest.mark.parametrize( + "fips_enabled, sha256_broken, enforces", + [(True, False, True), (False, False, False), (True, True, False)], + ids=["fips", "fips-flag-off", "sha256-broken"], +) +def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( + monkeypatch, fips_enabled, sha256_broken, enforces +): monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) - assert cryptography_enforces_fips() is fips_enabled + if sha256_broken: + monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) + assert cryptography_enforces_fips() is enforces def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): @@ -78,7 +89,7 @@ def test_this_runners_openssl_does_not_enforce_fips(): ("true", "false", True, True, True, "TLS certificate verification is disabled"), ("true", None, False, True, True, "TLS certificate verification is disabled"), ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), - ("true", None, True, True, False, "cryptography library's OpenSSL does not enforce FIPS"), + ("true", None, True, True, False, "cryptography library's OpenSSL has no working FIPS provider"), ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], ids=[ From 28c9a0d577894364c08ce8d9c440de1ec96c5f59 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 18:03:13 -0700 Subject: [PATCH 4/5] fix(proxy): refuse TLS verification off anywhere in FIPS mode A model's own ssl_verify: false reached the HTTP client after the startup check passed. get_ssl_verify now raises in FIPS mode whenever verification resolves to off, which also covers the startup check. --- litellm/llms/custom_httpx/http_handler.py | 10 ++++++++++ litellm/proxy/common_utils/fips.py | 6 +----- tests/unit/proxy/common_utils/test_fips.py | 8 ++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0b64885df45..ae9aa8f19d2 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -370,6 +370,16 @@ def get_ssl_verify( if ssl_cert_file and os.path.exists(ssl_cert_file): return ssl_cert_file + if ssl_verify is False: + from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode + + if is_fips_mode(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " + "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a " + "CA bundle." + ) + return ssl_verify if ssl_verify is not None else True diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 671c023bbdb..2ffb17852eb 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -58,11 +58,7 @@ def enforce_fips_mode() -> None: from litellm.llms.custom_httpx.http_handler import get_ssl_verify - if get_ssl_verify() is False: - raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY or " - "litellm_settings.ssl_verify. Remove the override, or point ssl_verify at a CA bundle." - ) + get_ssl_verify() # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off if not openssl_enforces_fips(): raise FipsModeError( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index bd6834af536..e70fd2e3b46 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -8,6 +8,7 @@ from cryptography.hazmat.bindings._rust import openssl as rust_openssl from cryptography.hazmat.primitives import hashes import litellm +from litellm.llms.custom_httpx.http_handler import get_ssl_verify from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, @@ -118,3 +119,10 @@ def test_enforce_fips_mode( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() + + +@pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) +def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused): + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): + assert get_ssl_verify(ssl_verify=False) is False From 6103a8c60c16b2641d73eacd279ced26b6bf2f7e Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 18:33:09 -0700 Subject: [PATCH 5/5] refactor(proxy): return FIPS boot refusals as values and harden the FIPS tests --- litellm/llms/custom_httpx/http_handler.py | 25 ++++++---- litellm/proxy/common_utils/fips.py | 31 ++++++++---- .../configuration/test_fips_mode_boot.py | 4 +- tests/unit/proxy/common_utils/test_fips.py | 46 ++++++++++++------ ...test_proxy_server_endpoints_and_startup.py | 48 +++++++++++++++---- 5 files changed, 109 insertions(+), 45 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index ae9aa8f19d2..81cd624883b 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -336,6 +336,21 @@ def _create_ssl_context( def get_ssl_verify( ssl_verify: bool | str | None = None, +) -> bool | str: + """resolve_ssl_verify, refusing disabled TLS verification under LITELLM_FIPS_MODE.""" + resolved: Final = resolve_ssl_verify(ssl_verify) + if resolved is not False: + return resolved + + from litellm.proxy.common_utils.fips import TLS_VERIFICATION_DISABLED, FipsModeError, is_fips_mode + + if is_fips_mode(): + raise FipsModeError(TLS_VERIFICATION_DISABLED) + return resolved + + +def resolve_ssl_verify( + ssl_verify: bool | str | None = None, ) -> bool | str: """ Common utility to resolve the SSL verification setting. @@ -370,16 +385,6 @@ def get_ssl_verify( if ssl_cert_file and os.path.exists(ssl_cert_file): return ssl_cert_file - if ssl_verify is False: - from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode - - if is_fips_mode(): - raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " - "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a " - "CA bundle." - ) - return ssl_verify if ssl_verify is not None else True diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 2ffb17852eb..2cbe154d2a1 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -4,6 +4,10 @@ from typing import Final from litellm.secret_managers.main import str_to_bool FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +TLS_VERIFICATION_DISABLED: Final = ( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " + "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a CA bundle." +) class FipsModeError(Exception): @@ -47,26 +51,35 @@ def cryptography_enforces_fips() -> bool: return True -def enforce_fips_mode() -> None: - """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or Python's or the cryptography library's OpenSSL does not enforce FIPS.""" +def fips_mode_refusal() -> str | None: + """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off + or Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: - raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") + return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it." if not is_fips_mode(): - return + return None - from litellm.llms.custom_httpx.http_handler import get_ssl_verify + from litellm.llms.custom_httpx.http_handler import resolve_ssl_verify - get_ssl_verify() # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off + if resolve_ssl_verify() is False: + return TLS_VERIFICATION_DISABLED if not openssl_enforces_fips(): - raise FipsModeError( + return ( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." ) if not cryptography_enforces_fips(): - raise FipsModeError( + return ( f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " f"{FIPS_MODE_ENV_VAR}." ) + return None + + +def enforce_fips_mode() -> None: + """Raise FipsModeError with the fips_mode_refusal reason, if there is one.""" + refusal: Final = fips_mode_refusal() + if refusal is not None: + raise FipsModeError(refusal) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index 85c7011876a..0a20d14fb19 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -58,5 +58,7 @@ def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Ga def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) - with owned_proxy(gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config) as proxy: + with owned_proxy( + gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config, remove_environment=("LITELLM_FIPS_MODE",) + ) as proxy: assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index e70fd2e3b46..596d45f88d1 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,5 +1,6 @@ import _hashlib from contextlib import nullcontext +from typing import Final from unittest.mock import Mock import pytest @@ -18,20 +19,20 @@ from litellm.proxy.common_utils.fips import ( ) -def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: - real_new = _hashlib.new +def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new: Final = _hashlib.new - def new(name, data=b"", **kwargs): + def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: if name == "md5" and md5_refused: raise _hashlib.UnsupportedDigestmodError("unsupported hash type") - return real_new(name, data, **kwargs) + return real_new(name, data, usedforsecurity=usedforsecurity) monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) monkeypatch.setattr(_hashlib, "new", new) @pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) -def test_is_fips_mode(monkeypatch, value, expected): +def test_is_fips_mode(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool) -> None: if value is None: monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) else: @@ -44,12 +45,14 @@ def test_is_fips_mode(monkeypatch, value, expected): [(1, True, True), (0, True, False), (1, False, False)], ids=["fips", "fips-flag-off", "md5-allowed"], ) -def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(monkeypatch, fips_flag, md5_refused, enforces): +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5( + monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool +) -> None: _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) assert openssl_enforces_fips() is enforces -def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(_hashlib, "get_fips_mode") assert openssl_enforces_fips() is False @@ -60,23 +63,28 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): ids=["fips", "fips-flag-off", "sha256-broken"], ) def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( - monkeypatch, fips_enabled, sha256_broken, enforces -): + monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool +) -> None: monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) if sha256_broken: monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) assert cryptography_enforces_fips() is enforces -def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(rust_openssl, "is_fips_enabled") assert cryptography_enforces_fips() is False -def test_this_runners_openssl_does_not_enforce_fips(): +def test_this_runners_python_openssl_does_not_enforce_fips() -> None: if _hashlib.get_fips_mode() == 1: - pytest.skip("this runner's OpenSSL is in FIPS mode") + pytest.skip("this runner's Python OpenSSL is in FIPS mode") assert openssl_enforces_fips() is False + + +def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: + if rust_openssl.is_fips_enabled(): + pytest.skip("this runner's cryptography OpenSSL is in FIPS mode") assert cryptography_enforces_fips() is False @@ -106,8 +114,14 @@ def test_this_runners_openssl_does_not_enforce_fips(): ], ) def test_enforce_fips_mode( - monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal -): + monkeypatch: pytest.MonkeyPatch, + fips_mode: str | None, + ssl_verify_env: str | None, + ssl_verify_setting: bool, + openssl_fips: bool, + cryptography_fips: bool, + refusal: str | None, +) -> None: for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): if value is None: monkeypatch.delenv(name, raising=False) @@ -122,7 +136,9 @@ def test_enforce_fips_mode( @pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) -def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused): +def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, refused: bool +) -> None: monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): assert get_ssl_verify(ssl_verify=False) is False diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 031c772dc44..4bdadbba02e 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1776,7 +1776,9 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.proxy_server import proxy_startup_event @@ -1791,22 +1793,48 @@ async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_veri pass +class _UsersWithOnePlaintextPassword: + async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: + return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) + + async def update(self, where: object, data: object) -> None: + raise AssertionError("a refused boot must not write to the database") + + +class _DatabaseWithOnePlaintextPassword: + def __init__(self) -> None: + self.db: Final = types.SimpleNamespace(litellm_usertable=_UsersWithOnePlaintextPassword()) + + @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(monkeypatch, tmp_path): +async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import _hashlib + import hashlib + + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + from litellm.proxy.common_utils.fips import FipsModeError - from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_startup_event + from litellm.proxy.proxy_server import proxy_startup_event + + real_new: Final = _hashlib.new + + def new_refusing_md5(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: + if name == "md5": + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, usedforsecurity=usedforsecurity) _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) monkeypatch.setattr(litellm, "ssl_verify", True) monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) - monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) - monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) - scrypt_refused = ValueError("[digital envelope routines] unsupported") - monkeypatch.setattr( - "litellm.proxy.proxy_server.migrate_passwords_to_scrypt_async", AsyncMock(side_effect=scrypt_refused) - ) + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr(_hashlib, "new", new_refusing_md5) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + scrypt_refused: Final = ValueError("[digital envelope routines] unsupported") + monkeypatch.setattr(hashlib, "scrypt", mock.Mock(side_effect=scrypt_refused)) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _DatabaseWithOnePlaintextPassword()) with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: async with proxy_startup_event(FastAPI()):