From db7c9aa5469d0e1e8f09cc959be7c6efa06e6eb8 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 10:31:54 -0700 Subject: [PATCH 01/14] feat(proxy): add LITELLM_FIPS_MODE startup gate With LITELLM_FIPS_MODE=true the proxy refuses to start when TLS verification is off, when Python's OpenSSL does not enforce FIPS, or when the startup password migration fails. A value that is not true/false also refuses. --- litellm/proxy/common_utils/fips.py | 53 ++++++++++++ litellm/proxy/proxy_server.py | 10 ++- .../configuration/test_fips_mode_boot.py | 62 ++++++++++++++ tests/unit/proxy/common_utils/test_fips.py | 85 +++++++++++++++++++ ...test_proxy_server_endpoints_and_startup.py | 39 +++++++++ 5 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/common_utils/fips.py create mode 100644 tests/integration/configuration/test_fips_mode_boot.py create mode 100644 tests/unit/proxy/common_utils/test_fips.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..3a8df10b9e5 --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,53 @@ +import os +from typing import Final + +from litellm.secret_managers.main import str_to_bool + +FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" + + +class FipsModeError(Exception): + pass + + +def is_fips_mode() -> bool: + return str_to_bool(os.getenv(FIPS_MODE_ENV_VAR)) is True + + +def openssl_enforces_fips() -> bool: + """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" + try: + import _hashlib + + if _hashlib.get_fips_mode() != 1: + return False + except (ImportError, AttributeError, ValueError): + return False + try: + _hashlib.new("md5", b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def enforce_fips_mode() -> None: + """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off + or OpenSSL does not enforce FIPS.""" + value: Final = os.getenv(FIPS_MODE_ENV_VAR) + if value and str_to_bool(value) is None: + raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") + if not is_fips_mode(): + return + + from litellm.llms.custom_httpx.http_handler import get_ssl_verify + + if get_ssl_verify() is False: + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY or " + "litellm_settings.ssl_verify. Remove the override, or point ssl_verify at a CA bundle." + ) + if not openssl_enforces_fips(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " + f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bcba76ca1b3..5aacc7ad1e6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -489,6 +489,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1523,6 +1524,8 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState if decoded_worker_config is not None: await initialize_from_worker_config(decoded_worker_config) + enforce_fips_mode() + enforce_master_key_boot_verdict( await with_stored_secrets_counted( master_key_boot_verdict( @@ -1565,9 +1568,14 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState result: Final = await migrate_passwords_to_scrypt_async(prisma_client) verbose_proxy_logger.info("Password migration: %s", result) except Exception as e: + if is_fips_mode(): + raise FipsModeError(f"Password migration failed, so plaintext passwords stay unhashed: {e}") from e verbose_proxy_logger.warning("Password migration skipped: %s", e) - asyncio.create_task(_run_pw_migration()) + if is_fips_mode(): + await _run_pw_migration() + else: + asyncio.create_task(_run_pw_migration()) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..85c7011876a --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,62 @@ +import _hashlib +import json +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway +from tests.integration._support.process import owned_proxy, refused_boot_log + + +def _config(directory: Path, upstream_url: str, *, ssl_verify: bool = True) -> Path: + config: Final = directory / "fips_boot.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "fips-model", + "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, + } + ], + "litellm_settings": {"ssl_verify": ssl_verify}, + } + ) + ) + return config + + +def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + log: Final = refused_boot_log( + gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=_config(tmp_path, gateway.upstream_url) + ) + assert "LITELLM_FIPS_MODE is on but OpenSSL does not enforce FIPS" in log, log + + +@pytest.mark.parametrize( + "overrides, ssl_verify", + [({"SSL_VERIFY": "false"}, True), ({}, False)], + ids=["ssl-verify-env", "litellm-settings"], +) +def test_fips_mode_refuses_to_start_with_tls_verification_disabled( + gateway: Gateway, tmp_path: Path, overrides: dict[str, str], ssl_verify: bool +) -> None: + config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=ssl_verify) + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", **overrides}, config=config) + assert "LITELLM_FIPS_MODE is on but TLS certificate verification is disabled" in log, log + + +def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log( + gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) + ) + assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log + + +def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: + config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) + with owned_proxy(gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config) as proxy: + assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..392db0d9cb4 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -0,0 +1,85 @@ +import _hashlib +from contextlib import nullcontext + +import pytest + +import litellm +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode, openssl_enforces_fips + + +def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new = _hashlib.new + + def new(name, data=b"", **kwargs): + if name == "md5" and md5_refused: + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, **kwargs) + + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) + monkeypatch.setattr(_hashlib, "new", new) + + +@pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) +def test_is_fips_mode(monkeypatch, value, expected): + if value is None: + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + else: + monkeypatch.setenv("LITELLM_FIPS_MODE", value) + assert is_fips_mode() is expected + + +@pytest.mark.parametrize( + "fips_flag, md5_refused, enforces", + [(1, True, True), (0, True, False), (1, False, False)], + ids=["fips", "fips-flag-off", "md5-allowed"], +) +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(monkeypatch, fips_flag, md5_refused, enforces): + _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) + assert openssl_enforces_fips() is enforces + + +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): + monkeypatch.delattr(_hashlib, "get_fips_mode") + assert openssl_enforces_fips() is False + + +def test_this_runners_openssl_does_not_enforce_fips(): + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + assert openssl_enforces_fips() is False + + +@pytest.mark.parametrize( + "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal", + [ + (None, "false", False, False, None), + ("", "false", False, False, None), + ("false", "false", False, False, None), + ("true", None, True, True, None), + ("true", "false", True, True, "TLS certificate verification is disabled"), + ("true", None, False, True, "TLS certificate verification is disabled"), + ("true", None, True, False, "OpenSSL does not enforce FIPS"), + ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + ], + ids=[ + "unset", + "empty", + "false", + "fips", + "ssl-verify-env-off", + "ssl-verify-setting-off", + "openssl-not-fips", + "not-a-boolean", + ], +) +def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal): + for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) + monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) + _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + enforce_fips_mode() diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 745488bf93f..12f0161ab0c 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1775,6 +1775,45 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + config_path, _ = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + config_path.write_text(yaml.dump({"litellm_settings": {"ssl_verify": False}})) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError, match="TLS certificate verification is disabled"): + async with proxy_startup_event(FastAPI()): + pass + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(monkeypatch, tmp_path): + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) + monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) + scrypt_refused = ValueError("[digital envelope routines] unsupported") + monkeypatch.setattr( + "litellm.proxy.proxy_server.migrate_passwords_to_scrypt_async", AsyncMock(side_effect=scrypt_refused) + ) + + with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: + async with proxy_startup_event(FastAPI()): + pass + + assert refused.value.__cause__ is scrypt_refused + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext From 75c6d07a8f87d7beb92e4304be8da24bf5046aa2 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 14:17:56 -0700 Subject: [PATCH 02/14] feat(proxy): refuse FIPS mode when cryptography's OpenSSL is not FIPS The stock cryptography wheel bundles its own non-FIPS OpenSSL, which serves JWT, Fernet and AES-GCM even when Python's OpenSSL enforces FIPS. --- litellm/proxy/common_utils/fips.py | 17 ++++++- tests/unit/proxy/common_utils/test_fips.py | 46 ++++++++++++++----- ...test_proxy_server_endpoints_and_startup.py | 1 + 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 3a8df10b9e5..f57f56d3248 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -30,9 +30,19 @@ def openssl_enforces_fips() -> bool: return False +def cryptography_enforces_fips() -> bool: + """The cryptography library's own OpenSSL runs with FIPS-only defaults.""" + try: + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + + return rust_openssl.is_fips_enabled() + except (ImportError, AttributeError): + return False + + def enforce_fips_mode() -> None: """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or OpenSSL does not enforce FIPS.""" + or Python's or the cryptography library's OpenSSL does not enforce FIPS.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") @@ -51,3 +61,8 @@ def enforce_fips_mode() -> None: f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." ) + if not cryptography_enforces_fips(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL does not enforce FIPS. Build it against " + f"the system OpenSSL (pip install --no-binary cryptography cryptography), or unset {FIPS_MODE_ENV_VAR}." + ) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 392db0d9cb4..98e95263dcb 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -2,9 +2,16 @@ import _hashlib from contextlib import nullcontext import pytest +from cryptography.hazmat.bindings._rust import openssl as rust_openssl import litellm -from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode, openssl_enforces_fips +from litellm.proxy.common_utils.fips import ( + FipsModeError, + cryptography_enforces_fips, + enforce_fips_mode, + is_fips_mode, + openssl_enforces_fips, +) def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: @@ -43,23 +50,36 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): assert openssl_enforces_fips() is False +@pytest.mark.parametrize("fips_enabled", [True, False]) +def test_cryptography_enforces_fips_reads_its_openssl_fips_flag(monkeypatch, fips_enabled): + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) + assert cryptography_enforces_fips() is fips_enabled + + +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): + monkeypatch.delattr(rust_openssl, "is_fips_enabled") + assert cryptography_enforces_fips() is False + + def test_this_runners_openssl_does_not_enforce_fips(): if _hashlib.get_fips_mode() == 1: pytest.skip("this runner's OpenSSL is in FIPS mode") assert openssl_enforces_fips() is False + assert cryptography_enforces_fips() is False @pytest.mark.parametrize( - "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal", + "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal", [ - (None, "false", False, False, None), - ("", "false", False, False, None), - ("false", "false", False, False, None), - ("true", None, True, True, None), - ("true", "false", True, True, "TLS certificate verification is disabled"), - ("true", None, False, True, "TLS certificate verification is disabled"), - ("true", None, True, False, "OpenSSL does not enforce FIPS"), - ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + (None, "false", False, False, False, None), + ("", "false", False, False, False, None), + ("false", "false", False, False, False, None), + ("true", None, True, True, True, None), + ("true", "false", True, True, True, "TLS certificate verification is disabled"), + ("true", None, False, True, True, "TLS certificate verification is disabled"), + ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), + ("true", None, True, True, False, "cryptography library's OpenSSL does not enforce FIPS"), + ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], ids=[ "unset", @@ -69,10 +89,13 @@ def test_this_runners_openssl_does_not_enforce_fips(): "ssl-verify-env-off", "ssl-verify-setting-off", "openssl-not-fips", + "cryptography-not-fips", "not-a-boolean", ], ) -def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal): +def test_enforce_fips_mode( + monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal +): for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): if value is None: monkeypatch.delenv(name, raising=False) @@ -80,6 +103,7 @@ def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_se monkeypatch.setenv(name, value) monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 12f0161ab0c..031c772dc44 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1801,6 +1801,7 @@ async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) + monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) scrypt_refused = ValueError("[digital envelope routines] unsupported") monkeypatch.setattr( From 23ebd158bf15861b102afd86e0edb090b54e450e Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 16:33:22 -0700 Subject: [PATCH 03/14] fix(proxy): refuse FIPS mode when cryptography's FIPS provider cannot hash The stock cryptography wheel reads the host's FIPS config and reports FIPS on, but its bundled OpenSSL cannot load the host's FIPS provider, so every operation fails. Require a working SHA-256 alongside the FIPS flag. --- litellm/proxy/common_utils/fips.py | 20 ++++++++++++++------ tests/unit/proxy/common_utils/test_fips.py | 19 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index f57f56d3248..671c023bbdb 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -31,13 +31,20 @@ def openssl_enforces_fips() -> bool: def cryptography_enforces_fips() -> bool: - """The cryptography library's own OpenSSL runs with FIPS-only defaults.""" + """The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256.""" try: + from cryptography.exceptions import InternalError, UnsupportedAlgorithm from cryptography.hazmat.bindings._rust import openssl as rust_openssl - - return rust_openssl.is_fips_enabled() - except (ImportError, AttributeError): + from cryptography.hazmat.primitives import hashes + except ImportError: return False + try: + if not rust_openssl.is_fips_enabled(): + return False + hashes.Hash(hashes.SHA256()).finalize() + except (AttributeError, InternalError, UnsupportedAlgorithm): + return False + return True def enforce_fips_mode() -> None: @@ -63,6 +70,7 @@ def enforce_fips_mode() -> None: ) if not cryptography_enforces_fips(): raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL does not enforce FIPS. Build it against " - f"the system OpenSSL (pip install --no-binary cryptography cryptography), or unset {FIPS_MODE_ENV_VAR}." + f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " + "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " + f"{FIPS_MODE_ENV_VAR}." ) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 98e95263dcb..bd6834af536 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,8 +1,11 @@ import _hashlib from contextlib import nullcontext +from unittest.mock import Mock import pytest +from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes import litellm from litellm.proxy.common_utils.fips import ( @@ -50,10 +53,18 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): assert openssl_enforces_fips() is False -@pytest.mark.parametrize("fips_enabled", [True, False]) -def test_cryptography_enforces_fips_reads_its_openssl_fips_flag(monkeypatch, fips_enabled): +@pytest.mark.parametrize( + "fips_enabled, sha256_broken, enforces", + [(True, False, True), (False, False, False), (True, True, False)], + ids=["fips", "fips-flag-off", "sha256-broken"], +) +def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( + monkeypatch, fips_enabled, sha256_broken, enforces +): monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) - assert cryptography_enforces_fips() is fips_enabled + if sha256_broken: + monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) + assert cryptography_enforces_fips() is enforces def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): @@ -78,7 +89,7 @@ def test_this_runners_openssl_does_not_enforce_fips(): ("true", "false", True, True, True, "TLS certificate verification is disabled"), ("true", None, False, True, True, "TLS certificate verification is disabled"), ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), - ("true", None, True, True, False, "cryptography library's OpenSSL does not enforce FIPS"), + ("true", None, True, True, False, "cryptography library's OpenSSL has no working FIPS provider"), ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], ids=[ From 28c9a0d577894364c08ce8d9c440de1ec96c5f59 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 18:03:13 -0700 Subject: [PATCH 04/14] fix(proxy): refuse TLS verification off anywhere in FIPS mode A model's own ssl_verify: false reached the HTTP client after the startup check passed. get_ssl_verify now raises in FIPS mode whenever verification resolves to off, which also covers the startup check. --- litellm/llms/custom_httpx/http_handler.py | 10 ++++++++++ litellm/proxy/common_utils/fips.py | 6 +----- tests/unit/proxy/common_utils/test_fips.py | 8 ++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0b64885df45..ae9aa8f19d2 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -370,6 +370,16 @@ def get_ssl_verify( if ssl_cert_file and os.path.exists(ssl_cert_file): return ssl_cert_file + if ssl_verify is False: + from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode + + if is_fips_mode(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " + "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a " + "CA bundle." + ) + return ssl_verify if ssl_verify is not None else True diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 671c023bbdb..2ffb17852eb 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -58,11 +58,7 @@ def enforce_fips_mode() -> None: from litellm.llms.custom_httpx.http_handler import get_ssl_verify - if get_ssl_verify() is False: - raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY or " - "litellm_settings.ssl_verify. Remove the override, or point ssl_verify at a CA bundle." - ) + get_ssl_verify() # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off if not openssl_enforces_fips(): raise FipsModeError( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index bd6834af536..e70fd2e3b46 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -8,6 +8,7 @@ from cryptography.hazmat.bindings._rust import openssl as rust_openssl from cryptography.hazmat.primitives import hashes import litellm +from litellm.llms.custom_httpx.http_handler import get_ssl_verify from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, @@ -118,3 +119,10 @@ def test_enforce_fips_mode( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() + + +@pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) +def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused): + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): + assert get_ssl_verify(ssl_verify=False) is False From 6103a8c60c16b2641d73eacd279ced26b6bf2f7e Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 18:33:09 -0700 Subject: [PATCH 05/14] refactor(proxy): return FIPS boot refusals as values and harden the FIPS tests --- litellm/llms/custom_httpx/http_handler.py | 25 ++++++---- litellm/proxy/common_utils/fips.py | 31 ++++++++---- .../configuration/test_fips_mode_boot.py | 4 +- tests/unit/proxy/common_utils/test_fips.py | 46 ++++++++++++------ ...test_proxy_server_endpoints_and_startup.py | 48 +++++++++++++++---- 5 files changed, 109 insertions(+), 45 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index ae9aa8f19d2..81cd624883b 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -336,6 +336,21 @@ def _create_ssl_context( def get_ssl_verify( ssl_verify: bool | str | None = None, +) -> bool | str: + """resolve_ssl_verify, refusing disabled TLS verification under LITELLM_FIPS_MODE.""" + resolved: Final = resolve_ssl_verify(ssl_verify) + if resolved is not False: + return resolved + + from litellm.proxy.common_utils.fips import TLS_VERIFICATION_DISABLED, FipsModeError, is_fips_mode + + if is_fips_mode(): + raise FipsModeError(TLS_VERIFICATION_DISABLED) + return resolved + + +def resolve_ssl_verify( + ssl_verify: bool | str | None = None, ) -> bool | str: """ Common utility to resolve the SSL verification setting. @@ -370,16 +385,6 @@ def get_ssl_verify( if ssl_cert_file and os.path.exists(ssl_cert_file): return ssl_cert_file - if ssl_verify is False: - from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode - - if is_fips_mode(): - raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " - "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a " - "CA bundle." - ) - return ssl_verify if ssl_verify is not None else True diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 2ffb17852eb..2cbe154d2a1 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -4,6 +4,10 @@ from typing import Final from litellm.secret_managers.main import str_to_bool FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +TLS_VERIFICATION_DISABLED: Final = ( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " + "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a CA bundle." +) class FipsModeError(Exception): @@ -47,26 +51,35 @@ def cryptography_enforces_fips() -> bool: return True -def enforce_fips_mode() -> None: - """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or Python's or the cryptography library's OpenSSL does not enforce FIPS.""" +def fips_mode_refusal() -> str | None: + """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off + or Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: - raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") + return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it." if not is_fips_mode(): - return + return None - from litellm.llms.custom_httpx.http_handler import get_ssl_verify + from litellm.llms.custom_httpx.http_handler import resolve_ssl_verify - get_ssl_verify() # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off + if resolve_ssl_verify() is False: + return TLS_VERIFICATION_DISABLED if not openssl_enforces_fips(): - raise FipsModeError( + return ( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." ) if not cryptography_enforces_fips(): - raise FipsModeError( + return ( f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " f"{FIPS_MODE_ENV_VAR}." ) + return None + + +def enforce_fips_mode() -> None: + """Raise FipsModeError with the fips_mode_refusal reason, if there is one.""" + refusal: Final = fips_mode_refusal() + if refusal is not None: + raise FipsModeError(refusal) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index 85c7011876a..0a20d14fb19 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -58,5 +58,7 @@ def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Ga def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) - with owned_proxy(gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config) as proxy: + with owned_proxy( + gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config, remove_environment=("LITELLM_FIPS_MODE",) + ) as proxy: assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index e70fd2e3b46..596d45f88d1 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,5 +1,6 @@ import _hashlib from contextlib import nullcontext +from typing import Final from unittest.mock import Mock import pytest @@ -18,20 +19,20 @@ from litellm.proxy.common_utils.fips import ( ) -def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: - real_new = _hashlib.new +def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new: Final = _hashlib.new - def new(name, data=b"", **kwargs): + def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: if name == "md5" and md5_refused: raise _hashlib.UnsupportedDigestmodError("unsupported hash type") - return real_new(name, data, **kwargs) + return real_new(name, data, usedforsecurity=usedforsecurity) monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) monkeypatch.setattr(_hashlib, "new", new) @pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) -def test_is_fips_mode(monkeypatch, value, expected): +def test_is_fips_mode(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool) -> None: if value is None: monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) else: @@ -44,12 +45,14 @@ def test_is_fips_mode(monkeypatch, value, expected): [(1, True, True), (0, True, False), (1, False, False)], ids=["fips", "fips-flag-off", "md5-allowed"], ) -def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(monkeypatch, fips_flag, md5_refused, enforces): +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5( + monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool +) -> None: _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) assert openssl_enforces_fips() is enforces -def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(_hashlib, "get_fips_mode") assert openssl_enforces_fips() is False @@ -60,23 +63,28 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): ids=["fips", "fips-flag-off", "sha256-broken"], ) def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( - monkeypatch, fips_enabled, sha256_broken, enforces -): + monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool +) -> None: monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) if sha256_broken: monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) assert cryptography_enforces_fips() is enforces -def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(rust_openssl, "is_fips_enabled") assert cryptography_enforces_fips() is False -def test_this_runners_openssl_does_not_enforce_fips(): +def test_this_runners_python_openssl_does_not_enforce_fips() -> None: if _hashlib.get_fips_mode() == 1: - pytest.skip("this runner's OpenSSL is in FIPS mode") + pytest.skip("this runner's Python OpenSSL is in FIPS mode") assert openssl_enforces_fips() is False + + +def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: + if rust_openssl.is_fips_enabled(): + pytest.skip("this runner's cryptography OpenSSL is in FIPS mode") assert cryptography_enforces_fips() is False @@ -106,8 +114,14 @@ def test_this_runners_openssl_does_not_enforce_fips(): ], ) def test_enforce_fips_mode( - monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal -): + monkeypatch: pytest.MonkeyPatch, + fips_mode: str | None, + ssl_verify_env: str | None, + ssl_verify_setting: bool, + openssl_fips: bool, + cryptography_fips: bool, + refusal: str | None, +) -> None: for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): if value is None: monkeypatch.delenv(name, raising=False) @@ -122,7 +136,9 @@ def test_enforce_fips_mode( @pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) -def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused): +def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, refused: bool +) -> None: monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): assert get_ssl_verify(ssl_verify=False) is False diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 031c772dc44..4bdadbba02e 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1776,7 +1776,9 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.proxy_server import proxy_startup_event @@ -1791,22 +1793,48 @@ async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_veri pass +class _UsersWithOnePlaintextPassword: + async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: + return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) + + async def update(self, where: object, data: object) -> None: + raise AssertionError("a refused boot must not write to the database") + + +class _DatabaseWithOnePlaintextPassword: + def __init__(self) -> None: + self.db: Final = types.SimpleNamespace(litellm_usertable=_UsersWithOnePlaintextPassword()) + + @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(monkeypatch, tmp_path): +async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import _hashlib + import hashlib + + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + from litellm.proxy.common_utils.fips import FipsModeError - from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_startup_event + from litellm.proxy.proxy_server import proxy_startup_event + + real_new: Final = _hashlib.new + + def new_refusing_md5(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: + if name == "md5": + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, usedforsecurity=usedforsecurity) _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) monkeypatch.setattr(litellm, "ssl_verify", True) monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) - monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) - monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) - scrypt_refused = ValueError("[digital envelope routines] unsupported") - monkeypatch.setattr( - "litellm.proxy.proxy_server.migrate_passwords_to_scrypt_async", AsyncMock(side_effect=scrypt_refused) - ) + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr(_hashlib, "new", new_refusing_md5) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + scrypt_refused: Final = ValueError("[digital envelope routines] unsupported") + monkeypatch.setattr(hashlib, "scrypt", mock.Mock(side_effect=scrypt_refused)) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _DatabaseWithOnePlaintextPassword()) with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: async with proxy_startup_event(FastAPI()): From bd24880ee5b46f6229b44b9a8d9a79c642630171 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 30 Oct 2026 18:03:09 -0700 Subject: [PATCH 06/14] fix(proxy): stop refusing TLS verification off in FIPS mode Skipping certificate verification is a security choice, not a FIPS one: the TLS connection still only uses algorithms the FIPS provider allows. --- litellm/llms/custom_httpx/http_handler.py | 15 ------- litellm/proxy/common_utils/fips.py | 13 +------ .../configuration/test_fips_mode_boot.py | 26 +------------ tests/unit/proxy/common_utils/test_fips.py | 39 ++++++------------- ...test_proxy_server_endpoints_and_startup.py | 20 ---------- 5 files changed, 16 insertions(+), 97 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 81cd624883b..0b64885df45 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -336,21 +336,6 @@ def _create_ssl_context( def get_ssl_verify( ssl_verify: bool | str | None = None, -) -> bool | str: - """resolve_ssl_verify, refusing disabled TLS verification under LITELLM_FIPS_MODE.""" - resolved: Final = resolve_ssl_verify(ssl_verify) - if resolved is not False: - return resolved - - from litellm.proxy.common_utils.fips import TLS_VERIFICATION_DISABLED, FipsModeError, is_fips_mode - - if is_fips_mode(): - raise FipsModeError(TLS_VERIFICATION_DISABLED) - return resolved - - -def resolve_ssl_verify( - ssl_verify: bool | str | None = None, ) -> bool | str: """ Common utility to resolve the SSL verification setting. diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 2cbe154d2a1..155696ea70f 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -4,10 +4,6 @@ from typing import Final from litellm.secret_managers.main import str_to_bool FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" -TLS_VERIFICATION_DISABLED: Final = ( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " - "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a CA bundle." -) class FipsModeError(Exception): @@ -52,18 +48,13 @@ def cryptography_enforces_fips() -> bool: def fips_mode_refusal() -> str | None: - """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" + """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but Python's or the + cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it." if not is_fips_mode(): return None - - from litellm.llms.custom_httpx.http_handler import resolve_ssl_verify - - if resolve_ssl_verify() is False: - return TLS_VERIFICATION_DISABLED if not openssl_enforces_fips(): return ( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index 0a20d14fb19..e2d7975b252 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -6,10 +6,10 @@ from typing import Final import pytest from tests.integration._support.client import Gateway -from tests.integration._support.process import owned_proxy, refused_boot_log +from tests.integration._support.process import refused_boot_log -def _config(directory: Path, upstream_url: str, *, ssl_verify: bool = True) -> Path: +def _config(directory: Path, upstream_url: str) -> Path: config: Final = directory / "fips_boot.yaml" config.write_text( json.dumps( @@ -20,7 +20,6 @@ def _config(directory: Path, upstream_url: str, *, ssl_verify: bool = True) -> P "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, } ], - "litellm_settings": {"ssl_verify": ssl_verify}, } ) ) @@ -36,29 +35,8 @@ def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: assert "LITELLM_FIPS_MODE is on but OpenSSL does not enforce FIPS" in log, log -@pytest.mark.parametrize( - "overrides, ssl_verify", - [({"SSL_VERIFY": "false"}, True), ({}, False)], - ids=["ssl-verify-env", "litellm-settings"], -) -def test_fips_mode_refuses_to_start_with_tls_verification_disabled( - gateway: Gateway, tmp_path: Path, overrides: dict[str, str], ssl_verify: bool -) -> None: - config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=ssl_verify) - log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", **overrides}, config=config) - assert "LITELLM_FIPS_MODE is on but TLS certificate verification is disabled" in log, log - - def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: log: Final = refused_boot_log( gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) ) assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log - - -def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: - config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) - with owned_proxy( - gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config, remove_environment=("LITELLM_FIPS_MODE",) - ) as proxy: - assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 596d45f88d1..a2afea25306 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -8,8 +8,6 @@ from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._rust import openssl as rust_openssl from cryptography.hazmat.primitives import hashes -import litellm -from litellm.llms.custom_httpx.http_handler import get_ssl_verify from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, @@ -89,25 +87,23 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: @pytest.mark.parametrize( - "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal", + "fips_mode, ssl_verify, openssl_fips, cryptography_fips, refusal", [ - (None, "false", False, False, False, None), - ("", "false", False, False, False, None), - ("false", "false", False, False, False, None), - ("true", None, True, True, True, None), - ("true", "false", True, True, True, "TLS certificate verification is disabled"), - ("true", None, False, True, True, "TLS certificate verification is disabled"), - ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), - ("true", None, True, True, False, "cryptography library's OpenSSL has no working FIPS provider"), - ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + (None, None, False, False, None), + ("", None, False, False, None), + ("false", None, False, False, None), + ("true", None, True, True, None), + ("true", "false", True, True, None), + ("true", None, False, True, "is on but OpenSSL does not enforce FIPS"), + ("true", None, True, False, "cryptography library's OpenSSL has no working FIPS provider"), + ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], ids=[ "unset", "empty", "false", "fips", - "ssl-verify-env-off", - "ssl-verify-setting-off", + "fips-with-tls-verification-off", "openssl-not-fips", "cryptography-not-fips", "not-a-boolean", @@ -116,29 +112,18 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: def test_enforce_fips_mode( monkeypatch: pytest.MonkeyPatch, fips_mode: str | None, - ssl_verify_env: str | None, - ssl_verify_setting: bool, + ssl_verify: str | None, openssl_fips: bool, cryptography_fips: bool, refusal: str | None, ) -> None: - for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): + for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify)): if value is None: monkeypatch.delenv(name, raising=False) else: monkeypatch.setenv(name, value) - monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() - - -@pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) -def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode( - monkeypatch: pytest.MonkeyPatch, fips_mode: str, refused: bool -) -> None: - monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) - with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): - assert get_ssl_verify(ssl_verify=False) is False diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 4bdadbba02e..ea238f74076 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1775,24 +1775,6 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database -@pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from litellm.proxy.common_utils.fips import FipsModeError - from litellm.proxy.proxy_server import proxy_startup_event - - config_path, _ = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) - config_path.write_text(yaml.dump({"litellm_settings": {"ssl_verify": False}})) - monkeypatch.setattr(litellm, "ssl_verify", True) - monkeypatch.delenv("SSL_VERIFY", raising=False) - monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - - with pytest.raises(FipsModeError, match="TLS certificate verification is disabled"): - async with proxy_startup_event(FastAPI()): - pass - - class _UsersWithOnePlaintextPassword: async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) @@ -1826,8 +1808,6 @@ async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails return real_new(name, data, usedforsecurity=usedforsecurity) _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) - monkeypatch.setattr(litellm, "ssl_verify", True) - monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) monkeypatch.setattr(_hashlib, "new", new_refusing_md5) From f770015279d88b5f9ca32e4f75b9c0c79a8fe487 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 30 Oct 2026 19:33:15 -0700 Subject: [PATCH 07/14] feat(fips): refuse a database used without FIPS mode, drop the password migration change FIPS mode marks a new database with a fips_mode row in LiteLLM_Config and refuses one that already holds LiteLLM rows without it. Without FIPS mode, the proxy refuses a database carrying the marker. --- litellm/proxy/common_utils/fips.py | 49 ++++++++ litellm/proxy/proxy_server.py | 15 +-- litellm/repositories/fips_repository.py | 65 ++++++++++ .../configuration/test_fips_mode_boot.py | 25 ++++ .../database/test_fips_database.py | 93 ++++++++++++++ tests/unit/proxy/common_utils/test_fips.py | 116 ++++++++++++++---- ...test_proxy_server_endpoints_and_startup.py | 41 ++----- 7 files changed, 342 insertions(+), 62 deletions(-) create mode 100644 litellm/repositories/fips_repository.py create mode 100644 tests/integration/database/test_fips_database.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 155696ea70f..8c70b7d7825 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -1,6 +1,8 @@ import os from typing import Final +from litellm._logging import verbose_proxy_logger +from litellm.repositories.fips_repository import FIPS_MARKER, FipsRepository from litellm.secret_managers.main import str_to_bool FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" @@ -74,3 +76,50 @@ def enforce_fips_mode() -> None: refusal: Final = fips_mode_refusal() if refusal is not None: raise FipsModeError(refusal) + + +async def _is_marked_fips(database: FipsRepository) -> bool: + try: + return await database.is_marked_fips() + except Exception as error: # noqa: BLE001 # without FIPS mode, an unreadable database starts as it did before + verbose_proxy_logger.warning("Could not check whether the database was set up in FIPS mode: %s", error) + return False + + +async def fips_database_refusal(database: FipsRepository | None, *, database_is_configured: bool) -> str | None: + """Why the proxy must not start on its database: LITELLM_FIPS_MODE is off on a database set up in FIPS mode, or + on over a database that can't be checked or already holds rows written without it. None when it may start.""" + if not is_fips_mode(): + if database is not None and await _is_marked_fips(database): + return ( + f"This database was set up with {FIPS_MODE_ENV_VAR}=true, so the proxy only starts on it in FIPS mode. " + f"To stop using FIPS mode on it for good, run " + f"DELETE FROM \"LiteLLM_Config\" WHERE param_name = '{FIPS_MARKER}'." + ) + return None + if database is None: + return ( + f"{FIPS_MODE_ENV_VAR} is on but the database can't be reached to check it." + if database_is_configured + else None + ) + try: + state: Final = await database.state() + except Exception as error: # noqa: BLE001 # a value, so FIPS mode refuses a database it can't check + return f"{FIPS_MODE_ENV_VAR} is on but the database could not be checked: {error}" + if state.holds_rows and not state.marked_fips: + return ( + f"{FIPS_MODE_ENV_VAR} is on but this database already holds data written without it. " + "FIPS mode only starts on a new, empty database." + ) + return None + + +async def enforce_fips_database(database: FipsRepository | None, *, database_is_configured: bool) -> None: + """Raise FipsModeError with the fips_database_refusal reason, if there is one, then mark the database as set up + in FIPS mode when LITELLM_FIPS_MODE is on.""" + refusal: Final = await fips_database_refusal(database, database_is_configured=database_is_configured) + if refusal is not None: + raise FipsModeError(refusal) + if database is not None and is_fips_mode(): + await database.mark_fips() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 974eba6ca9d..57e36b0b62e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -499,7 +499,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id -from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode +from litellm.proxy.common_utils.fips import enforce_fips_database, enforce_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -992,6 +992,7 @@ from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.credentials_repository import CredentialsRepository +from litellm.repositories.fips_repository import FipsRepository from litellm.repositories.prisma_protocols import TableActions from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router import ( @@ -1640,6 +1641,11 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener database: Final = existing_database or await _own_database(resources, prisma_client) + await enforce_fips_database( + None if prisma_client is None else FipsRepository(prisma_client), + database_is_configured=get_secret("DATABASE_URL", None) is not None, + ) + await migrate_if_requested( environ=os.environ, master_key=master_key, @@ -1656,14 +1662,9 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener result: Final = await migrate_passwords_to_scrypt_async(startup_prisma) verbose_proxy_logger.info("Password migration: %s", result) except Exception as e: - if is_fips_mode(): - raise FipsModeError(f"Password migration failed, so plaintext passwords stay unhashed: {e}") from e verbose_proxy_logger.warning("Password migration skipped: %s", e) - if is_fips_mode(): - await _run_pw_migration() - else: - await resources.enter_async_context(database_runtime.background_task(_run_pw_migration())) + await resources.enter_async_context(database_runtime.background_task(_run_pw_migration())) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/litellm/repositories/fips_repository.py b/litellm/repositories/fips_repository.py new file mode 100644 index 00000000000..23a756e44bf --- /dev/null +++ b/litellm/repositories/fips_repository.py @@ -0,0 +1,65 @@ +"""FIPS repository for the LiteLLM_Config row that marks a database as set up in FIPS mode.""" + +from dataclasses import dataclass +from typing import Final, Protocol, cast # noqa: TID251 # Prisma delegates query_raw and execute_raw dynamically + +from litellm.proxy.db.create_views import SupportsRawQueries + +FIPS_MARKER: Final = "fips_mode" +_IS_MARKED: Final = 'SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1' +_OTHER_TABLES: Final = ( + "SELECT table_name FROM information_schema.tables WHERE table_schema = ANY (current_schemas(false)) " + "AND table_type = 'BASE TABLE' AND table_name LIKE 'LiteLLM\\_%' AND table_name <> 'LiteLLM_Config'" +) +_MARK: Final = ( + "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ($1, 'true'::jsonb) " + "ON CONFLICT (param_name) DO NOTHING" +) + + +def _quoted(identifier: str) -> str: + return '"' + identifier.replace('"', '""') + '"' + + +class _WithWriter(Protocol): + @property + def writer_db(self) -> object: ... + + +@dataclass(frozen=True, slots=True) +class FipsDatabaseState: + marked_fips: bool + holds_rows: bool + + +class FipsRepository: + """Reads and writes go to the writer, since a replica may not have the marker yet.""" + + def __init__(self, prisma_client: _WithWriter) -> None: + self._prisma_client: Final = prisma_client + + @property + def _writer(self) -> SupportsRawQueries: + writer: Final = self._prisma_client.writer_db + return cast(SupportsRawQueries, writer) # cast-ok: Prisma delegates database methods dynamically + + async def is_marked_fips(self) -> bool: + return bool(await self._writer.query_raw(_IS_MARKED, FIPS_MARKER)) + + async def state(self) -> FipsDatabaseState: + """The marker and whether any other LiteLLM row exists, read in one statement so they agree.""" + writer: Final = self._writer + tables: Final = tuple(str(row["table_name"]) for row in await writer.query_raw(_OTHER_TABLES)) + any_row_in_other_tables: Final = "".join(f" OR EXISTS (SELECT 1 FROM {_quoted(table)})" for table in tables) + row: Final = ( + await writer.query_raw( + 'SELECT EXISTS (SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1) AS marked_fips, ' + f'(EXISTS (SELECT 1 FROM "LiteLLM_Config" WHERE param_name <> $1){any_row_in_other_tables}) ' + "AS holds_rows", + FIPS_MARKER, + ) + )[0] + return FipsDatabaseState(marked_fips=row["marked_fips"] is True, holds_rows=row["holds_rows"] is True) + + async def mark_fips(self) -> None: + await self._writer.execute_raw(_MARK, FIPS_MARKER) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index e2d7975b252..11e1f8fc088 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -6,6 +6,7 @@ from typing import Final import pytest from tests.integration._support.client import Gateway +from tests.integration._support.database import scratch_database, write_rows from tests.integration._support.process import refused_boot_log @@ -40,3 +41,27 @@ def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Ga gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) ) assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log + + +def test_proxy_refuses_to_start_without_fips_mode_on_a_database_set_up_in_fips_mode( + gateway: Gateway, tmp_path: Path +) -> None: + with scratch_database() as database_url: + write_rows( + 'CREATE TABLE "LiteLLM_Config" (param_name TEXT NOT NULL PRIMARY KEY, param_value JSONB, ' + "last_run_at TIMESTAMP(3), reload_revision BIGINT NOT NULL DEFAULT 0)", + (), + database_url=database_url, + ) + write_rows( + "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('fips_mode', 'true')", + (), + database_url=database_url, + ) + log: Final = refused_boot_log( + gateway, + tmp_path, + {"LITELLM_FIPS_MODE": "false", "DATABASE_URL": database_url}, + config=_config(tmp_path, gateway.upstream_url), + ) + assert "This database was set up with LITELLM_FIPS_MODE=true" in log, log diff --git a/tests/integration/database/test_fips_database.py b/tests/integration/database/test_fips_database.py new file mode 100644 index 00000000000..21f86308fe2 --- /dev/null +++ b/tests/integration/database/test_fips_database.py @@ -0,0 +1,93 @@ +import asyncio +from contextlib import nullcontext +from typing import Final, LiteralString + +import pytest + +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_database +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.fips_repository import FipsRepository +from tests.integration._support.database import read_rows, scratch_database, write_rows + +_CONFIG_TABLE: Final = ( + 'CREATE TABLE "LiteLLM_Config" (param_name TEXT PRIMARY KEY, param_value JSONB, last_run_at TIMESTAMP(3), ' + "reload_revision BIGINT NOT NULL DEFAULT 0)" +) +_KEY_TABLE: Final = 'CREATE TABLE "LiteLLM_VerificationToken" (token TEXT PRIMARY KEY)' +_MARKER: Final = "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('fips_mode', 'true')" +_KEY: Final = "INSERT INTO \"LiteLLM_VerificationToken\" (token) VALUES ('hashed-key')" +_SETTING: Final = "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('general_settings', '{}')" + + +def _marker_rows(database_url: str) -> int: + return len( + read_rows('SELECT 1 FROM "LiteLLM_Config" WHERE param_name = %s', ("fips_mode",), database_url=database_url) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, setup, refusal, marked_after", + [ + ("true", (_CONFIG_TABLE, _KEY_TABLE), None, 1), + ("true", (_CONFIG_TABLE, _KEY_TABLE, _MARKER, _KEY), None, 1), + ("true", (_CONFIG_TABLE, _KEY_TABLE, _KEY), "already holds data written without it", 0), + ("true", (_CONFIG_TABLE, _KEY_TABLE, _SETTING), "already holds data written without it", 0), + ("true", (), "could not be checked", None), + ("false", (_CONFIG_TABLE, _KEY_TABLE, _MARKER), "set up with LITELLM_FIPS_MODE=true", 1), + ("false", (_CONFIG_TABLE, _KEY_TABLE, _KEY), None, 0), + ("false", (), None, None), + ], + ids=[ + "fips-on-new-database", + "fips-on-fips-database", + "fips-on-database-with-a-key", + "fips-on-database-with-a-setting", + "fips-on-database-without-tables", + "fips-off-fips-database", + "fips-off-database-with-a-key", + "fips-off-database-without-tables", + ], +) +async def test_fips_database_check( + monkeypatch: pytest.MonkeyPatch, + fips_mode: str, + setup: tuple[LiteralString, ...], + refusal: str | None, + marked_after: int | None, +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + with scratch_database() as database_url: + for statement in setup: + write_rows(statement, (), database_url=database_url) + monkeypatch.setenv("DATABASE_URL", database_url) + client: Final = PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(FipsRepository(client), database_is_configured=True) + finally: + await client.disconnect() + if marked_after is not None: + assert _marker_rows(database_url) == marked_after + + +@pytest.mark.asyncio +async def test_pods_starting_together_in_fips_mode_on_a_new_database_both_start( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + with scratch_database() as database_url: + for statement in (_CONFIG_TABLE, _KEY_TABLE): + write_rows(statement, (), database_url=database_url) + monkeypatch.setenv("DATABASE_URL", database_url) + pods: Final = tuple(PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) for _ in range(4)) + await asyncio.gather(*(pod.connect() for pod in pods)) + try: + await asyncio.gather( + *(enforce_fips_database(FipsRepository(pod), database_is_configured=True) for pod in pods) + ) + finally: + await asyncio.gather(*(pod.disconnect() for pod in pods)) + assert _marker_rows(database_url) == 1 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index a2afea25306..a48ba3138e2 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,4 +1,5 @@ import _hashlib +from collections.abc import Mapping, Sequence from contextlib import nullcontext from typing import Final from unittest.mock import Mock @@ -11,10 +12,12 @@ from cryptography.hazmat.primitives import hashes from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, + enforce_fips_database, enforce_fips_mode, is_fips_mode, openssl_enforces_fips, ) +from litellm.repositories.fips_repository import FipsRepository def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: @@ -87,43 +90,106 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: @pytest.mark.parametrize( - "fips_mode, ssl_verify, openssl_fips, cryptography_fips, refusal", + "fips_mode, openssl_fips, cryptography_fips, refusal", [ - (None, None, False, False, None), - ("", None, False, False, None), - ("false", None, False, False, None), - ("true", None, True, True, None), - ("true", "false", True, True, None), - ("true", None, False, True, "is on but OpenSSL does not enforce FIPS"), - ("true", None, True, False, "cryptography library's OpenSSL has no working FIPS provider"), - ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), - ], - ids=[ - "unset", - "empty", - "false", - "fips", - "fips-with-tls-verification-off", - "openssl-not-fips", - "cryptography-not-fips", - "not-a-boolean", + (None, False, False, None), + ("", False, False, None), + ("false", False, False, None), + ("true", True, True, None), + ("true", False, True, "is on but OpenSSL does not enforce FIPS"), + ("true", True, False, "cryptography library's OpenSSL has no working FIPS provider"), + ("enforced", True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], + ids=["unset", "empty", "false", "fips", "openssl-not-fips", "cryptography-not-fips", "not-a-boolean"], ) def test_enforce_fips_mode( monkeypatch: pytest.MonkeyPatch, fips_mode: str | None, - ssl_verify: str | None, openssl_fips: bool, cryptography_fips: bool, refusal: str | None, ) -> None: - for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify)): - if value is None: - monkeypatch.delenv(name, raising=False) - else: - monkeypatch.setenv(name, value) + if fips_mode is None: + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + else: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() + + +class _Database: + """Prisma's raw-query edge over a database whose only LiteLLM table besides LiteLLM_Config is the key table.""" + + def __init__(self, *, marked_fips: bool, holds_rows: bool, readable: bool = True) -> None: + self.marked_fips = marked_fips + self.holds_rows = holds_rows + self.readable = readable + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + if not self.readable: + raise RuntimeError('relation "LiteLLM_Config" does not exist') + if "information_schema.tables" in query: + return [{"table_name": "LiteLLM_VerificationToken"}] + if '"LiteLLM_VerificationToken"' in query: + return [{"marked_fips": self.marked_fips, "holds_rows": self.holds_rows}] + return [{"?column?": 1}] if self.marked_fips else [] + + async def execute_raw(self, query: str, *args: object) -> int: + self.marked_fips = True + return 1 + + +class _Prisma: + def __init__(self, database: _Database) -> None: + self.writer_db: Final = database + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, database, refusal, marked_after", + [ + ("true", _Database(marked_fips=False, holds_rows=False), None, True), + ("true", _Database(marked_fips=True, holds_rows=True), None, True), + ("true", _Database(marked_fips=False, holds_rows=True), "already holds data written without it", False), + ("true", _Database(marked_fips=False, holds_rows=False, readable=False), "could not be checked", False), + ("false", _Database(marked_fips=True, holds_rows=True), "set up with LITELLM_FIPS_MODE=true", True), + ("false", _Database(marked_fips=False, holds_rows=True), None, False), + ("false", _Database(marked_fips=True, holds_rows=True, readable=False), None, True), + ], + ids=[ + "fips-on-new-database", + "fips-on-fips-database", + "fips-on-used-database", + "fips-on-unreadable-database", + "fips-off-fips-database", + "fips-off-used-database", + "fips-off-unreadable-database", + ], +) +async def test_enforce_fips_database( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, database: _Database, refusal: str | None, marked_after: bool +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(FipsRepository(_Prisma(database)), database_is_configured=True) + + assert database.marked_fips is marked_after + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, database_is_configured, refusal", + [("true", True, "can't be reached"), ("true", False, None), ("false", True, None)], + ids=["fips-on-unreachable", "fips-on-no-database", "fips-off-unreachable"], +) +async def test_enforce_fips_database_without_a_connected_database( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, database_is_configured: bool, refusal: str | None +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(None, database_is_configured=database_is_configured) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 6ae5c47bfbc..18574e08a3e 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10,7 +10,7 @@ import subprocess import sys import time import types -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -1790,56 +1790,37 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database -class _UsersWithOnePlaintextPassword: - async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: - return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) +class _DatabaseSetUpInFipsMode: + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + return [{"?column?": 1}] - async def update(self, where: object, data: object) -> None: + async def execute_raw(self, query: str, *args: object) -> int: raise AssertionError("a refused boot must not write to the database") -class _DatabaseWithOnePlaintextPassword: +class _PrismaClientOnAFipsDatabase: def __init__(self) -> None: - self.db: Final = types.SimpleNamespace(litellm_usertable=_UsersWithOnePlaintextPassword()) + self.writer_db: Final = _DatabaseSetUpInFipsMode() async def aclose(self) -> None: pass @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails( +async def test_proxy_startup_refuses_a_database_set_up_in_fips_mode_when_fips_mode_is_off( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - import _hashlib - import hashlib - - from cryptography.hazmat.bindings._rust import openssl as rust_openssl - from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.proxy_server import proxy_startup_event - real_new: Final = _hashlib.new - - def new_refusing_md5(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: - if name == "md5": - raise _hashlib.UnsupportedDigestmodError("unsupported hash type") - return real_new(name, data, usedforsecurity=usedforsecurity) - _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) - monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) - monkeypatch.setattr(_hashlib, "new", new_refusing_md5) - monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) - scrypt_refused: Final = ValueError("[digital envelope routines] unsupported") - monkeypatch.setattr(hashlib, "scrypt", mock.Mock(side_effect=scrypt_refused)) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _DatabaseWithOnePlaintextPassword()) + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _PrismaClientOnAFipsDatabase()) - with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: + with pytest.raises(FipsModeError, match="set up with LITELLM_FIPS_MODE=true"): async with proxy_startup_event(FastAPI()): pass - assert refused.value.__cause__ is scrypt_refused - class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): From ba6f2b840d4d1b529e937edb48a24b06d391f690 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 30 Oct 2026 19:59:19 -0700 Subject: [PATCH 08/14] test(fips): close the database test clients with aclose so the engine stops --- tests/integration/database/test_fips_database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/database/test_fips_database.py b/tests/integration/database/test_fips_database.py index 21f86308fe2..54238bc6876 100644 --- a/tests/integration/database/test_fips_database.py +++ b/tests/integration/database/test_fips_database.py @@ -68,7 +68,7 @@ async def test_fips_database_check( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): await enforce_fips_database(FipsRepository(client), database_is_configured=True) finally: - await client.disconnect() + await client.aclose() if marked_after is not None: assert _marker_rows(database_url) == marked_after @@ -89,5 +89,5 @@ async def test_pods_starting_together_in_fips_mode_on_a_new_database_both_start( *(enforce_fips_database(FipsRepository(pod), database_is_configured=True) for pod in pods) ) finally: - await asyncio.gather(*(pod.disconnect() for pod in pods)) + await asyncio.gather(*(pod.aclose() for pod in pods)) assert _marker_rows(database_url) == 1 From 6569e2c132c64a11026a75075889a3ecb6edf398 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 00:58:36 -0700 Subject: [PATCH 09/14] feat(fips): read FIPS mode from general_settings.fips_mode instead of LITELLM_FIPS_MODE --- litellm/proxy/_types.py | 7 ++ litellm/proxy/common_utils/fips.py | 80 ++++++++++-------- litellm/proxy/proxy_server.py | 5 +- .../configuration/test_fips_mode_boot.py | 21 ++--- .../database/test_fips_database.py | 27 +++--- tests/unit/proxy/common_utils/test_fips.py | 84 ++++++++++--------- ...test_proxy_server_endpoints_and_startup.py | 18 +++- 7 files changed, 142 insertions(+), 100 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 07588053959..191209f03a0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2507,6 +2507,13 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="local development only: start even when master_key is unset, empty, or a publicly known default", ) + fips_mode: bool | None = Field( + None, + description=( + "refuse to start unless Python's and the cryptography library's OpenSSL enforce FIPS, and only start on " + "a new database or one set up in FIPS mode" + ), + ) coordination_redis: CoordinationRedisParams | None = Field( None, description=( diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 8c70b7d7825..9f895bf1a1d 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -1,19 +1,28 @@ -import os +from collections.abc import Mapping from typing import Final from litellm._logging import verbose_proxy_logger from litellm.repositories.fips_repository import FIPS_MARKER, FipsRepository from litellm.secret_managers.main import str_to_bool -FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +FIPS_MODE_SETTING: Final = "fips_mode" +_SETTING: Final = f"general_settings.{FIPS_MODE_SETTING}" class FipsModeError(Exception): pass -def is_fips_mode() -> bool: - return str_to_bool(os.getenv(FIPS_MODE_ENV_VAR)) is True +def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: + """general_settings.fips_mode as a boolean: False when unset, None when it is not a boolean.""" + value: Final = general_settings.get(FIPS_MODE_SETTING) + if value is None or isinstance(value, bool): + return value is True + return str_to_bool(value) if isinstance(value, str) else None + + +def is_fips_mode(general_settings: Mapping[str, object]) -> bool: + return _fips_mode_setting(general_settings) is True def openssl_enforces_fips() -> bool: @@ -49,31 +58,34 @@ def cryptography_enforces_fips() -> bool: return True -def fips_mode_refusal() -> str | None: - """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but Python's or the +def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None: + """Why the proxy must not start: general_settings.fips_mode is not a boolean, or is on but Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" - value: Final = os.getenv(FIPS_MODE_ENV_VAR) - if value and str_to_bool(value) is None: - return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it." - if not is_fips_mode(): + setting: Final = _fips_mode_setting(general_settings) + if setting is None: + return ( + f"{_SETTING}={general_settings.get(FIPS_MODE_SETTING)!r} is not a boolean. Set it to true or false, " + "or remove it." + ) + if not setting: return None if not openssl_enforces_fips(): return ( - f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " - f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." + f"{_SETTING} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with its FIPS " + f"provider active and default_properties = fips=yes, or turn {_SETTING} off." ) if not cryptography_enforces_fips(): return ( - f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " - "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " - f"{FIPS_MODE_ENV_VAR}." + f"{_SETTING} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " + "against the system OpenSSL (pip install --no-binary cryptography cryptography), or turn " + f"{_SETTING} off." ) return None -def enforce_fips_mode() -> None: +def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: """Raise FipsModeError with the fips_mode_refusal reason, if there is one.""" - refusal: Final = fips_mode_refusal() + refusal: Final = fips_mode_refusal(general_settings) if refusal is not None: raise FipsModeError(refusal) @@ -86,40 +98,42 @@ async def _is_marked_fips(database: FipsRepository) -> bool: return False -async def fips_database_refusal(database: FipsRepository | None, *, database_is_configured: bool) -> str | None: - """Why the proxy must not start on its database: LITELLM_FIPS_MODE is off on a database set up in FIPS mode, or - on over a database that can't be checked or already holds rows written without it. None when it may start.""" - if not is_fips_mode(): +async def fips_database_refusal( + database: FipsRepository | None, *, fips_mode: bool, database_is_configured: bool +) -> str | None: + """Why the proxy must not start on its database: FIPS mode is off on a database set up in FIPS mode, or on over + a database that can't be checked or already holds rows written without it. None when it may start.""" + if not fips_mode: if database is not None and await _is_marked_fips(database): return ( - f"This database was set up with {FIPS_MODE_ENV_VAR}=true, so the proxy only starts on it in FIPS mode. " + f"This database was set up with {_SETTING}: true, so the proxy only starts on it in FIPS mode. " f"To stop using FIPS mode on it for good, run " f"DELETE FROM \"LiteLLM_Config\" WHERE param_name = '{FIPS_MARKER}'." ) return None if database is None: - return ( - f"{FIPS_MODE_ENV_VAR} is on but the database can't be reached to check it." - if database_is_configured - else None - ) + return f"{_SETTING} is on but the database can't be reached to check it." if database_is_configured else None try: state: Final = await database.state() except Exception as error: # noqa: BLE001 # a value, so FIPS mode refuses a database it can't check - return f"{FIPS_MODE_ENV_VAR} is on but the database could not be checked: {error}" + return f"{_SETTING} is on but the database could not be checked: {error}" if state.holds_rows and not state.marked_fips: return ( - f"{FIPS_MODE_ENV_VAR} is on but this database already holds data written without it. " + f"{_SETTING} is on but this database already holds data written without it. " "FIPS mode only starts on a new, empty database." ) return None -async def enforce_fips_database(database: FipsRepository | None, *, database_is_configured: bool) -> None: +async def enforce_fips_database( + database: FipsRepository | None, *, fips_mode: bool, database_is_configured: bool +) -> None: """Raise FipsModeError with the fips_database_refusal reason, if there is one, then mark the database as set up - in FIPS mode when LITELLM_FIPS_MODE is on.""" - refusal: Final = await fips_database_refusal(database, database_is_configured=database_is_configured) + in FIPS mode when FIPS mode is on.""" + refusal: Final = await fips_database_refusal( + database, fips_mode=fips_mode, database_is_configured=database_is_configured + ) if refusal is not None: raise FipsModeError(refusal) - if database is not None and is_fips_mode(): + if database is not None and fips_mode: await database.mark_fips() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d02707cb55d..7bf5e07e4e6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -499,7 +499,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id -from litellm.proxy.common_utils.fips import enforce_fips_database, enforce_fips_mode +from litellm.proxy.common_utils.fips import enforce_fips_database, enforce_fips_mode, is_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1607,7 +1607,7 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener if decoded_worker_config is not None: await initialize_from_worker_config(decoded_worker_config) - enforce_fips_mode() + enforce_fips_mode(general_settings_view()) async with AsyncExitStack() as boot_database: enforce_master_key_boot_verdict( @@ -1643,6 +1643,7 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener await enforce_fips_database( None if prisma_client is None else FipsRepository(prisma_client), + fips_mode=is_fips_mode(general_settings_view()), database_is_configured=get_secret("DATABASE_URL", None) is not None, ) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index 11e1f8fc088..b4637043800 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -10,7 +10,7 @@ from tests.integration._support.database import scratch_database, write_rows from tests.integration._support.process import refused_boot_log -def _config(directory: Path, upstream_url: str) -> Path: +def _config(directory: Path, upstream_url: str, fips_mode: object) -> Path: config: Final = directory / "fips_boot.yaml" config.write_text( json.dumps( @@ -21,6 +21,7 @@ def _config(directory: Path, upstream_url: str) -> Path: "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, } ], + "general_settings": {"fips_mode": fips_mode}, } ) ) @@ -30,17 +31,13 @@ def _config(directory: Path, upstream_url: str) -> Path: def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: if _hashlib.get_fips_mode() == 1: pytest.skip("this runner's OpenSSL is in FIPS mode") - log: Final = refused_boot_log( - gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=_config(tmp_path, gateway.upstream_url) - ) - assert "LITELLM_FIPS_MODE is on but OpenSSL does not enforce FIPS" in log, log + log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, True)) + assert "general_settings.fips_mode is on but OpenSSL does not enforce FIPS" in log, log def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: - log: Final = refused_boot_log( - gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) - ) - assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log + log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, "enforced")) + assert "general_settings.fips_mode='enforced' is not a boolean" in log, log def test_proxy_refuses_to_start_without_fips_mode_on_a_database_set_up_in_fips_mode( @@ -61,7 +58,7 @@ def test_proxy_refuses_to_start_without_fips_mode_on_a_database_set_up_in_fips_m log: Final = refused_boot_log( gateway, tmp_path, - {"LITELLM_FIPS_MODE": "false", "DATABASE_URL": database_url}, - config=_config(tmp_path, gateway.upstream_url), + {"DATABASE_URL": database_url}, + config=_config(tmp_path, gateway.upstream_url, False), ) - assert "This database was set up with LITELLM_FIPS_MODE=true" in log, log + assert "This database was set up with general_settings.fips_mode: true" in log, log diff --git a/tests/integration/database/test_fips_database.py b/tests/integration/database/test_fips_database.py index 54238bc6876..36074e48719 100644 --- a/tests/integration/database/test_fips_database.py +++ b/tests/integration/database/test_fips_database.py @@ -30,14 +30,14 @@ def _marker_rows(database_url: str) -> int: @pytest.mark.parametrize( "fips_mode, setup, refusal, marked_after", [ - ("true", (_CONFIG_TABLE, _KEY_TABLE), None, 1), - ("true", (_CONFIG_TABLE, _KEY_TABLE, _MARKER, _KEY), None, 1), - ("true", (_CONFIG_TABLE, _KEY_TABLE, _KEY), "already holds data written without it", 0), - ("true", (_CONFIG_TABLE, _KEY_TABLE, _SETTING), "already holds data written without it", 0), - ("true", (), "could not be checked", None), - ("false", (_CONFIG_TABLE, _KEY_TABLE, _MARKER), "set up with LITELLM_FIPS_MODE=true", 1), - ("false", (_CONFIG_TABLE, _KEY_TABLE, _KEY), None, 0), - ("false", (), None, None), + (True, (_CONFIG_TABLE, _KEY_TABLE), None, 1), + (True, (_CONFIG_TABLE, _KEY_TABLE, _MARKER, _KEY), None, 1), + (True, (_CONFIG_TABLE, _KEY_TABLE, _KEY), "already holds data written without it", 0), + (True, (_CONFIG_TABLE, _KEY_TABLE, _SETTING), "already holds data written without it", 0), + (True, (), "could not be checked", None), + (False, (_CONFIG_TABLE, _KEY_TABLE, _MARKER), "set up with general_settings.fips_mode: true", 1), + (False, (_CONFIG_TABLE, _KEY_TABLE, _KEY), None, 0), + (False, (), None, None), ], ids=[ "fips-on-new-database", @@ -52,12 +52,11 @@ def _marker_rows(database_url: str) -> int: ) async def test_fips_database_check( monkeypatch: pytest.MonkeyPatch, - fips_mode: str, + fips_mode: bool, setup: tuple[LiteralString, ...], refusal: str | None, marked_after: int | None, ) -> None: - monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) with scratch_database() as database_url: for statement in setup: write_rows(statement, (), database_url=database_url) @@ -66,7 +65,7 @@ async def test_fips_database_check( await client.connect() try: with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): - await enforce_fips_database(FipsRepository(client), database_is_configured=True) + await enforce_fips_database(FipsRepository(client), fips_mode=fips_mode, database_is_configured=True) finally: await client.aclose() if marked_after is not None: @@ -77,7 +76,6 @@ async def test_fips_database_check( async def test_pods_starting_together_in_fips_mode_on_a_new_database_both_start( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setenv("LITELLM_FIPS_MODE", "true") with scratch_database() as database_url: for statement in (_CONFIG_TABLE, _KEY_TABLE): write_rows(statement, (), database_url=database_url) @@ -86,7 +84,10 @@ async def test_pods_starting_together_in_fips_mode_on_a_new_database_both_start( await asyncio.gather(*(pod.connect() for pod in pods)) try: await asyncio.gather( - *(enforce_fips_database(FipsRepository(pod), database_is_configured=True) for pod in pods) + *( + enforce_fips_database(FipsRepository(pod), fips_mode=True, database_is_configured=True) + for pod in pods + ) ) finally: await asyncio.gather(*(pod.aclose() for pod in pods)) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index a48ba3138e2..cccf261fd5c 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -32,13 +32,19 @@ def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refuse monkeypatch.setattr(_hashlib, "new", new) -@pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) -def test_is_fips_mode(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool) -> None: - if value is None: - monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) - else: - monkeypatch.setenv("LITELLM_FIPS_MODE", value) - assert is_fips_mode() is expected +@pytest.mark.parametrize( + "general_settings, expected", + [ + ({"fips_mode": True}, True), + ({"fips_mode": " TRUE "}, True), + ({"fips_mode": False}, False), + ({"fips_mode": "false"}, False), + ({"fips_mode": None}, False), + ({}, False), + ], +) +def test_is_fips_mode(general_settings: Mapping[str, object], expected: bool) -> None: + assert is_fips_mode(general_settings) is expected @pytest.mark.parametrize( @@ -90,34 +96,40 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: @pytest.mark.parametrize( - "fips_mode, openssl_fips, cryptography_fips, refusal", + "general_settings, openssl_fips, cryptography_fips, refusal", [ - (None, False, False, None), - ("", False, False, None), - ("false", False, False, None), - ("true", True, True, None), - ("true", False, True, "is on but OpenSSL does not enforce FIPS"), - ("true", True, False, "cryptography library's OpenSSL has no working FIPS provider"), - ("enforced", True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + ({}, False, False, None), + ({"fips_mode": False}, False, False, None), + ({"fips_mode": True}, True, True, None), + ({"fips_mode": "true"}, True, True, None), + ({"fips_mode": True}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"), + ({"fips_mode": True}, True, False, "cryptography library's OpenSSL has no working FIPS provider"), + ({"fips_mode": "enforced"}, True, True, "general_settings.fips_mode='enforced' is not a boolean"), + ({"fips_mode": 1}, True, True, "general_settings.fips_mode=1 is not a boolean"), + ], + ids=[ + "unset", + "false", + "fips", + "fips-from-a-string", + "openssl-not-fips", + "cryptography-not-fips", + "not-a-boolean", + "a-number", ], - ids=["unset", "empty", "false", "fips", "openssl-not-fips", "cryptography-not-fips", "not-a-boolean"], ) def test_enforce_fips_mode( monkeypatch: pytest.MonkeyPatch, - fips_mode: str | None, + general_settings: Mapping[str, object], openssl_fips: bool, cryptography_fips: bool, refusal: str | None, ) -> None: - if fips_mode is None: - monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) - else: - monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): - enforce_fips_mode() + enforce_fips_mode(general_settings) class _Database: @@ -151,13 +163,13 @@ class _Prisma: @pytest.mark.parametrize( "fips_mode, database, refusal, marked_after", [ - ("true", _Database(marked_fips=False, holds_rows=False), None, True), - ("true", _Database(marked_fips=True, holds_rows=True), None, True), - ("true", _Database(marked_fips=False, holds_rows=True), "already holds data written without it", False), - ("true", _Database(marked_fips=False, holds_rows=False, readable=False), "could not be checked", False), - ("false", _Database(marked_fips=True, holds_rows=True), "set up with LITELLM_FIPS_MODE=true", True), - ("false", _Database(marked_fips=False, holds_rows=True), None, False), - ("false", _Database(marked_fips=True, holds_rows=True, readable=False), None, True), + (True, _Database(marked_fips=False, holds_rows=False), None, True), + (True, _Database(marked_fips=True, holds_rows=True), None, True), + (True, _Database(marked_fips=False, holds_rows=True), "already holds data written without it", False), + (True, _Database(marked_fips=False, holds_rows=False, readable=False), "could not be checked", False), + (False, _Database(marked_fips=True, holds_rows=True), "set up with general_settings.fips_mode: true", True), + (False, _Database(marked_fips=False, holds_rows=True), None, False), + (False, _Database(marked_fips=True, holds_rows=True, readable=False), None, True), ], ids=[ "fips-on-new-database", @@ -170,12 +182,10 @@ class _Prisma: ], ) async def test_enforce_fips_database( - monkeypatch: pytest.MonkeyPatch, fips_mode: str, database: _Database, refusal: str | None, marked_after: bool + fips_mode: bool, database: _Database, refusal: str | None, marked_after: bool ) -> None: - monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) - with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): - await enforce_fips_database(FipsRepository(_Prisma(database)), database_is_configured=True) + await enforce_fips_database(FipsRepository(_Prisma(database)), fips_mode=fips_mode, database_is_configured=True) assert database.marked_fips is marked_after @@ -183,13 +193,11 @@ async def test_enforce_fips_database( @pytest.mark.asyncio @pytest.mark.parametrize( "fips_mode, database_is_configured, refusal", - [("true", True, "can't be reached"), ("true", False, None), ("false", True, None)], + [(True, True, "can't be reached"), (True, False, None), (False, True, None)], ids=["fips-on-unreachable", "fips-on-no-database", "fips-off-unreachable"], ) async def test_enforce_fips_database_without_a_connected_database( - monkeypatch: pytest.MonkeyPatch, fips_mode: str, database_is_configured: bool, refusal: str | None + fips_mode: bool, database_is_configured: bool, refusal: str | None ) -> None: - monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) - with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): - await enforce_fips_database(None, database_is_configured=database_is_configured) + await enforce_fips_database(None, fips_mode=fips_mode, database_is_configured=database_is_configured) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 99e78ae6385..98658dbf731 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1814,10 +1814,24 @@ async def test_proxy_startup_refuses_a_database_set_up_in_fips_mode_when_fips_mo from litellm.proxy.proxy_server import proxy_startup_event _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) - monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _PrismaClientOnAFipsDatabase()) - with pytest.raises(FipsModeError, match="set up with LITELLM_FIPS_MODE=true"): + with pytest.raises(FipsModeError, match=r"set up with general_settings\.fips_mode: true"): + async with proxy_startup_event(FastAPI()): + pass + + +@pytest.mark.asyncio +async def test_proxy_startup_reads_fips_mode_from_the_config_file( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key", "fips_mode": True}) + monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: False) + + with pytest.raises(FipsModeError, match=r"general_settings\.fips_mode is on but OpenSSL does not enforce FIPS"): async with proxy_startup_event(FastAPI()): pass From e9216053387c148a8252cc12b67b8adcd8fb8af4 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 01:04:07 -0700 Subject: [PATCH 10/14] refactor(fips): keep fips_mode out of the API-writable general settings, it is read from the config file only --- litellm/proxy/_types.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 191209f03a0..07588053959 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2507,13 +2507,6 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="local development only: start even when master_key is unset, empty, or a publicly known default", ) - fips_mode: bool | None = Field( - None, - description=( - "refuse to start unless Python's and the cryptography library's OpenSSL enforce FIPS, and only start on " - "a new database or one set up in FIPS mode" - ), - ) coordination_redis: CoordinationRedisParams | None = Field( None, description=( From 08ee70545ff7effa8da086a5d055fc6a1e334f9b Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 01:43:04 -0700 Subject: [PATCH 11/14] fix(fips): retry the startup marker and state reads once after a dropped connection A dropped connection during the non-FIPS marker read used to count as an unmarked database, so a blip at boot let a non-FIPS proxy start on a FIPS database. Both reads now go through call_with_db_reconnect_retry, the reconnect-and-retry the other startup reads use. A read that still fails keeps its old outcome. --- litellm/repositories/fips_repository.py | 20 ++++++++++++++-- tests/unit/proxy/common_utils/test_fips.py | 28 +++++++++++++++++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/litellm/repositories/fips_repository.py b/litellm/repositories/fips_repository.py index 23a756e44bf..15c80272483 100644 --- a/litellm/repositories/fips_repository.py +++ b/litellm/repositories/fips_repository.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import Final, Protocol, cast # noqa: TID251 # Prisma delegates query_raw and execute_raw dynamically from litellm.proxy.db.create_views import SupportsRawQueries +from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry FIPS_MARKER: Final = "fips_mode" _IS_MARKED: Final = 'SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1' @@ -25,6 +26,10 @@ class _WithWriter(Protocol): @property def writer_db(self) -> object: ... + async def attempt_db_reconnect( + self, *, reason: str, timeout_seconds: float | None = None, lock_timeout_seconds: float | None = None + ) -> bool: ... + @dataclass(frozen=True, slots=True) class FipsDatabaseState: @@ -33,7 +38,8 @@ class FipsDatabaseState: class FipsRepository: - """Reads and writes go to the writer, since a replica may not have the marker yet.""" + """Reads and writes go to the writer, since a replica may not have the marker yet. A read that loses its + connection reconnects and runs once more.""" def __init__(self, prisma_client: _WithWriter) -> None: self._prisma_client: Final = prisma_client @@ -44,10 +50,20 @@ class FipsRepository: return cast(SupportsRawQueries, writer) # cast-ok: Prisma delegates database methods dynamically async def is_marked_fips(self) -> bool: - return bool(await self._writer.query_raw(_IS_MARKED, FIPS_MARKER)) + return await call_with_db_reconnect_retry( + self._prisma_client, self._read_marker, reason="fips_startup_read_marker_failure" + ) async def state(self) -> FipsDatabaseState: """The marker and whether any other LiteLLM row exists, read in one statement so they agree.""" + return await call_with_db_reconnect_retry( + self._prisma_client, self._read_state, reason="fips_startup_read_state_failure" + ) + + async def _read_marker(self) -> bool: + return bool(await self._writer.query_raw(_IS_MARKED, FIPS_MARKER)) + + async def _read_state(self) -> FipsDatabaseState: writer: Final = self._writer tables: Final = tuple(str(row["table_name"]) for row in await writer.query_raw(_OTHER_TABLES)) any_row_in_other_tables: Final = "".join(f" OR EXISTS (SELECT 1 FROM {_quoted(table)})" for table in tables) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index cccf261fd5c..732cfa55d75 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -4,6 +4,7 @@ from contextlib import nullcontext from typing import Final from unittest.mock import Mock +import httpx import pytest from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._rust import openssl as rust_openssl @@ -135,12 +136,16 @@ def test_enforce_fips_mode( class _Database: """Prisma's raw-query edge over a database whose only LiteLLM table besides LiteLLM_Config is the key table.""" - def __init__(self, *, marked_fips: bool, holds_rows: bool, readable: bool = True) -> None: + def __init__(self, *, marked_fips: bool, holds_rows: bool, readable: bool = True, dropped_reads: int = 0) -> None: self.marked_fips = marked_fips self.holds_rows = holds_rows self.readable = readable + self.dropped_reads = dropped_reads async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + if self.dropped_reads: + self.dropped_reads -= 1 + raise httpx.ReadError("connection dropped") if not self.readable: raise RuntimeError('relation "LiteLLM_Config" does not exist') if "information_schema.tables" in query: @@ -158,6 +163,11 @@ class _Prisma: def __init__(self, database: _Database) -> None: self.writer_db: Final = database + async def attempt_db_reconnect( + self, *, reason: str, timeout_seconds: float | None = None, lock_timeout_seconds: float | None = None + ) -> bool: + return True + @pytest.mark.asyncio @pytest.mark.parametrize( @@ -170,6 +180,19 @@ class _Prisma: (False, _Database(marked_fips=True, holds_rows=True), "set up with general_settings.fips_mode: true", True), (False, _Database(marked_fips=False, holds_rows=True), None, False), (False, _Database(marked_fips=True, holds_rows=True, readable=False), None, True), + ( + True, + _Database(marked_fips=False, holds_rows=True, dropped_reads=1), + "already holds data written without it", + False, + ), + ( + False, + _Database(marked_fips=True, holds_rows=True, dropped_reads=1), + "set up with general_settings.fips_mode: true", + True, + ), + (False, _Database(marked_fips=True, holds_rows=True, dropped_reads=2), None, True), ], ids=[ "fips-on-new-database", @@ -179,6 +202,9 @@ class _Prisma: "fips-off-fips-database", "fips-off-used-database", "fips-off-unreadable-database", + "fips-on-used-database-after-a-dropped-connection", + "fips-off-fips-database-after-a-dropped-connection", + "fips-off-fips-database-dropped-twice", ], ) async def test_enforce_fips_database( From c721edbfd6a4de23b815faa3861b9af4d1ed9765 Mon Sep 17 00:00:00 2001 From: yucheng Date: Sat, 31 Oct 2026 06:39:24 -0700 Subject: [PATCH 12/14] fix(fips): refuse a fips_mode key that has no value fips_mode: os.environ/X with X unset, or a bare fips_mode: / null, resolved to None and started without FIPS mode and without a message, while an empty string refused. A key that is present now has to be a boolean; only a config without the key means off. --- litellm/proxy/common_utils/fips.py | 22 ++++++++++++------- tests/unit/proxy/common_utils/test_fips.py | 2 ++ ...test_proxy_server_endpoints_and_startup.py | 17 ++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 9f895bf1a1d..0667143f53c 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -14,10 +14,13 @@ class FipsModeError(Exception): def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: - """general_settings.fips_mode as a boolean: False when unset, None when it is not a boolean.""" - value: Final = general_settings.get(FIPS_MODE_SETTING) - if value is None or isinstance(value, bool): - return value is True + """general_settings.fips_mode as a boolean: False when the key is absent, None when it is not a boolean, including + a key with no value.""" + if FIPS_MODE_SETTING not in general_settings: + return False + value: Final = general_settings[FIPS_MODE_SETTING] + if isinstance(value, bool): + return value return str_to_bool(value) if isinstance(value, str) else None @@ -63,10 +66,13 @@ def fips_mode_refusal(general_settings: Mapping[str, object]) -> str | None: cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" setting: Final = _fips_mode_setting(general_settings) if setting is None: - return ( - f"{_SETTING}={general_settings.get(FIPS_MODE_SETTING)!r} is not a boolean. Set it to true or false, " - "or remove it." - ) + value: Final = general_settings.get(FIPS_MODE_SETTING) + if value is None: + return ( + f"{_SETTING} has no value: it is empty, or an os.environ/ reference to a variable that is not set. " + "Set it to true or false, or remove it." + ) + return f"{_SETTING}={value!r} is not a boolean. Set it to true or false, or remove it." if not setting: return None if not openssl_enforces_fips(): diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 732cfa55d75..ae857993ed3 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -107,6 +107,7 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: ({"fips_mode": True}, True, False, "cryptography library's OpenSSL has no working FIPS provider"), ({"fips_mode": "enforced"}, True, True, "general_settings.fips_mode='enforced' is not a boolean"), ({"fips_mode": 1}, True, True, "general_settings.fips_mode=1 is not a boolean"), + ({"fips_mode": None}, True, True, "general_settings.fips_mode has no value"), ], ids=[ "unset", @@ -117,6 +118,7 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: "cryptography-not-fips", "not-a-boolean", "a-number", + "no-value", ], ) def test_enforce_fips_mode( diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 98658dbf731..c46c1b336ae 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1836,6 +1836,23 @@ async def test_proxy_startup_reads_fips_mode_from_the_config_file( pass +@pytest.mark.asyncio +async def test_proxy_startup_refuses_a_fips_mode_reference_to_an_unset_variable( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings( + monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key", "fips_mode": "os.environ/FIPS_MODE_NOT_SET"} + ) + monkeypatch.delenv("FIPS_MODE_NOT_SET", raising=False) + + with pytest.raises(FipsModeError, match=r"general_settings\.fips_mode has no value"): + async with proxy_startup_event(FastAPI()): + pass + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext From f91489a7cc534fd63ccf6aac0f3907f5e172341e Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 2 Nov 2026 12:03:44 -0800 Subject: [PATCH 13/14] refactor(fips): move the database check out of the boot gate The boot gate keeps the setting and the two OpenSSL checks. The check that a database matches the FIPS setting moves to its own PR, so a boot without the flag runs no extra database read. --- litellm/proxy/common_utils/fips.py | 55 -------- litellm/proxy/proxy_server.py | 9 +- litellm/repositories/fips_repository.py | 81 ------------ .../configuration/test_fips_mode_boot.py | 25 ---- .../database/test_fips_database.py | 94 -------------- tests/unit/proxy/common_utils/test_fips.py | 117 +----------------- ...test_proxy_server_endpoints_and_startup.py | 33 +---- 7 files changed, 3 insertions(+), 411 deletions(-) delete mode 100644 litellm/repositories/fips_repository.py delete mode 100644 tests/integration/database/test_fips_database.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 0667143f53c..37142364af9 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -1,8 +1,6 @@ from collections.abc import Mapping from typing import Final -from litellm._logging import verbose_proxy_logger -from litellm.repositories.fips_repository import FIPS_MARKER, FipsRepository from litellm.secret_managers.main import str_to_bool FIPS_MODE_SETTING: Final = "fips_mode" @@ -24,10 +22,6 @@ def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: return str_to_bool(value) if isinstance(value, str) else None -def is_fips_mode(general_settings: Mapping[str, object]) -> bool: - return _fips_mode_setting(general_settings) is True - - def openssl_enforces_fips() -> bool: """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" try: @@ -94,52 +88,3 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: refusal: Final = fips_mode_refusal(general_settings) if refusal is not None: raise FipsModeError(refusal) - - -async def _is_marked_fips(database: FipsRepository) -> bool: - try: - return await database.is_marked_fips() - except Exception as error: # noqa: BLE001 # without FIPS mode, an unreadable database starts as it did before - verbose_proxy_logger.warning("Could not check whether the database was set up in FIPS mode: %s", error) - return False - - -async def fips_database_refusal( - database: FipsRepository | None, *, fips_mode: bool, database_is_configured: bool -) -> str | None: - """Why the proxy must not start on its database: FIPS mode is off on a database set up in FIPS mode, or on over - a database that can't be checked or already holds rows written without it. None when it may start.""" - if not fips_mode: - if database is not None and await _is_marked_fips(database): - return ( - f"This database was set up with {_SETTING}: true, so the proxy only starts on it in FIPS mode. " - f"To stop using FIPS mode on it for good, run " - f"DELETE FROM \"LiteLLM_Config\" WHERE param_name = '{FIPS_MARKER}'." - ) - return None - if database is None: - return f"{_SETTING} is on but the database can't be reached to check it." if database_is_configured else None - try: - state: Final = await database.state() - except Exception as error: # noqa: BLE001 # a value, so FIPS mode refuses a database it can't check - return f"{_SETTING} is on but the database could not be checked: {error}" - if state.holds_rows and not state.marked_fips: - return ( - f"{_SETTING} is on but this database already holds data written without it. " - "FIPS mode only starts on a new, empty database." - ) - return None - - -async def enforce_fips_database( - database: FipsRepository | None, *, fips_mode: bool, database_is_configured: bool -) -> None: - """Raise FipsModeError with the fips_database_refusal reason, if there is one, then mark the database as set up - in FIPS mode when FIPS mode is on.""" - refusal: Final = await fips_database_refusal( - database, fips_mode=fips_mode, database_is_configured=database_is_configured - ) - if refusal is not None: - raise FipsModeError(refusal) - if database is not None and fips_mode: - await database.mark_fips() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7bf5e07e4e6..baaeced2375 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -499,7 +499,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id -from litellm.proxy.common_utils.fips import enforce_fips_database, enforce_fips_mode, is_fips_mode +from litellm.proxy.common_utils.fips import enforce_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -992,7 +992,6 @@ from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.credentials_repository import CredentialsRepository -from litellm.repositories.fips_repository import FipsRepository from litellm.repositories.prisma_protocols import TableActions from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router import ( @@ -1641,12 +1640,6 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener database: Final = existing_database or await _own_database(resources, prisma_client) - await enforce_fips_database( - None if prisma_client is None else FipsRepository(prisma_client), - fips_mode=is_fips_mode(general_settings_view()), - database_is_configured=get_secret("DATABASE_URL", None) is not None, - ) - await migrate_if_requested( environ=os.environ, master_key=master_key, diff --git a/litellm/repositories/fips_repository.py b/litellm/repositories/fips_repository.py deleted file mode 100644 index 15c80272483..00000000000 --- a/litellm/repositories/fips_repository.py +++ /dev/null @@ -1,81 +0,0 @@ -"""FIPS repository for the LiteLLM_Config row that marks a database as set up in FIPS mode.""" - -from dataclasses import dataclass -from typing import Final, Protocol, cast # noqa: TID251 # Prisma delegates query_raw and execute_raw dynamically - -from litellm.proxy.db.create_views import SupportsRawQueries -from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry - -FIPS_MARKER: Final = "fips_mode" -_IS_MARKED: Final = 'SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1' -_OTHER_TABLES: Final = ( - "SELECT table_name FROM information_schema.tables WHERE table_schema = ANY (current_schemas(false)) " - "AND table_type = 'BASE TABLE' AND table_name LIKE 'LiteLLM\\_%' AND table_name <> 'LiteLLM_Config'" -) -_MARK: Final = ( - "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ($1, 'true'::jsonb) " - "ON CONFLICT (param_name) DO NOTHING" -) - - -def _quoted(identifier: str) -> str: - return '"' + identifier.replace('"', '""') + '"' - - -class _WithWriter(Protocol): - @property - def writer_db(self) -> object: ... - - async def attempt_db_reconnect( - self, *, reason: str, timeout_seconds: float | None = None, lock_timeout_seconds: float | None = None - ) -> bool: ... - - -@dataclass(frozen=True, slots=True) -class FipsDatabaseState: - marked_fips: bool - holds_rows: bool - - -class FipsRepository: - """Reads and writes go to the writer, since a replica may not have the marker yet. A read that loses its - connection reconnects and runs once more.""" - - def __init__(self, prisma_client: _WithWriter) -> None: - self._prisma_client: Final = prisma_client - - @property - def _writer(self) -> SupportsRawQueries: - writer: Final = self._prisma_client.writer_db - return cast(SupportsRawQueries, writer) # cast-ok: Prisma delegates database methods dynamically - - async def is_marked_fips(self) -> bool: - return await call_with_db_reconnect_retry( - self._prisma_client, self._read_marker, reason="fips_startup_read_marker_failure" - ) - - async def state(self) -> FipsDatabaseState: - """The marker and whether any other LiteLLM row exists, read in one statement so they agree.""" - return await call_with_db_reconnect_retry( - self._prisma_client, self._read_state, reason="fips_startup_read_state_failure" - ) - - async def _read_marker(self) -> bool: - return bool(await self._writer.query_raw(_IS_MARKED, FIPS_MARKER)) - - async def _read_state(self) -> FipsDatabaseState: - writer: Final = self._writer - tables: Final = tuple(str(row["table_name"]) for row in await writer.query_raw(_OTHER_TABLES)) - any_row_in_other_tables: Final = "".join(f" OR EXISTS (SELECT 1 FROM {_quoted(table)})" for table in tables) - row: Final = ( - await writer.query_raw( - 'SELECT EXISTS (SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1) AS marked_fips, ' - f'(EXISTS (SELECT 1 FROM "LiteLLM_Config" WHERE param_name <> $1){any_row_in_other_tables}) ' - "AS holds_rows", - FIPS_MARKER, - ) - )[0] - return FipsDatabaseState(marked_fips=row["marked_fips"] is True, holds_rows=row["holds_rows"] is True) - - async def mark_fips(self) -> None: - await self._writer.execute_raw(_MARK, FIPS_MARKER) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index b4637043800..18fb692da6e 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -6,7 +6,6 @@ from typing import Final import pytest from tests.integration._support.client import Gateway -from tests.integration._support.database import scratch_database, write_rows from tests.integration._support.process import refused_boot_log @@ -38,27 +37,3 @@ def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, "enforced")) assert "general_settings.fips_mode='enforced' is not a boolean" in log, log - - -def test_proxy_refuses_to_start_without_fips_mode_on_a_database_set_up_in_fips_mode( - gateway: Gateway, tmp_path: Path -) -> None: - with scratch_database() as database_url: - write_rows( - 'CREATE TABLE "LiteLLM_Config" (param_name TEXT NOT NULL PRIMARY KEY, param_value JSONB, ' - "last_run_at TIMESTAMP(3), reload_revision BIGINT NOT NULL DEFAULT 0)", - (), - database_url=database_url, - ) - write_rows( - "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('fips_mode', 'true')", - (), - database_url=database_url, - ) - log: Final = refused_boot_log( - gateway, - tmp_path, - {"DATABASE_URL": database_url}, - config=_config(tmp_path, gateway.upstream_url, False), - ) - assert "This database was set up with general_settings.fips_mode: true" in log, log diff --git a/tests/integration/database/test_fips_database.py b/tests/integration/database/test_fips_database.py deleted file mode 100644 index 36074e48719..00000000000 --- a/tests/integration/database/test_fips_database.py +++ /dev/null @@ -1,94 +0,0 @@ -import asyncio -from contextlib import nullcontext -from typing import Final, LiteralString - -import pytest - -from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_database -from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache -from litellm.proxy.utils import PrismaClient, ProxyLogging -from litellm.repositories.fips_repository import FipsRepository -from tests.integration._support.database import read_rows, scratch_database, write_rows - -_CONFIG_TABLE: Final = ( - 'CREATE TABLE "LiteLLM_Config" (param_name TEXT PRIMARY KEY, param_value JSONB, last_run_at TIMESTAMP(3), ' - "reload_revision BIGINT NOT NULL DEFAULT 0)" -) -_KEY_TABLE: Final = 'CREATE TABLE "LiteLLM_VerificationToken" (token TEXT PRIMARY KEY)' -_MARKER: Final = "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('fips_mode', 'true')" -_KEY: Final = "INSERT INTO \"LiteLLM_VerificationToken\" (token) VALUES ('hashed-key')" -_SETTING: Final = "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('general_settings', '{}')" - - -def _marker_rows(database_url: str) -> int: - return len( - read_rows('SELECT 1 FROM "LiteLLM_Config" WHERE param_name = %s', ("fips_mode",), database_url=database_url) - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "fips_mode, setup, refusal, marked_after", - [ - (True, (_CONFIG_TABLE, _KEY_TABLE), None, 1), - (True, (_CONFIG_TABLE, _KEY_TABLE, _MARKER, _KEY), None, 1), - (True, (_CONFIG_TABLE, _KEY_TABLE, _KEY), "already holds data written without it", 0), - (True, (_CONFIG_TABLE, _KEY_TABLE, _SETTING), "already holds data written without it", 0), - (True, (), "could not be checked", None), - (False, (_CONFIG_TABLE, _KEY_TABLE, _MARKER), "set up with general_settings.fips_mode: true", 1), - (False, (_CONFIG_TABLE, _KEY_TABLE, _KEY), None, 0), - (False, (), None, None), - ], - ids=[ - "fips-on-new-database", - "fips-on-fips-database", - "fips-on-database-with-a-key", - "fips-on-database-with-a-setting", - "fips-on-database-without-tables", - "fips-off-fips-database", - "fips-off-database-with-a-key", - "fips-off-database-without-tables", - ], -) -async def test_fips_database_check( - monkeypatch: pytest.MonkeyPatch, - fips_mode: bool, - setup: tuple[LiteralString, ...], - refusal: str | None, - marked_after: int | None, -) -> None: - with scratch_database() as database_url: - for statement in setup: - write_rows(statement, (), database_url=database_url) - monkeypatch.setenv("DATABASE_URL", database_url) - client: Final = PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) - await client.connect() - try: - with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): - await enforce_fips_database(FipsRepository(client), fips_mode=fips_mode, database_is_configured=True) - finally: - await client.aclose() - if marked_after is not None: - assert _marker_rows(database_url) == marked_after - - -@pytest.mark.asyncio -async def test_pods_starting_together_in_fips_mode_on_a_new_database_both_start( - monkeypatch: pytest.MonkeyPatch, -) -> None: - with scratch_database() as database_url: - for statement in (_CONFIG_TABLE, _KEY_TABLE): - write_rows(statement, (), database_url=database_url) - monkeypatch.setenv("DATABASE_URL", database_url) - pods: Final = tuple(PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) for _ in range(4)) - await asyncio.gather(*(pod.connect() for pod in pods)) - try: - await asyncio.gather( - *( - enforce_fips_database(FipsRepository(pod), fips_mode=True, database_is_configured=True) - for pod in pods - ) - ) - finally: - await asyncio.gather(*(pod.aclose() for pod in pods)) - assert _marker_rows(database_url) == 1 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index ae857993ed3..e81577f79a4 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,10 +1,9 @@ import _hashlib -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from contextlib import nullcontext from typing import Final from unittest.mock import Mock -import httpx import pytest from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._rust import openssl as rust_openssl @@ -13,12 +12,9 @@ from cryptography.hazmat.primitives import hashes from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, - enforce_fips_database, enforce_fips_mode, - is_fips_mode, openssl_enforces_fips, ) -from litellm.repositories.fips_repository import FipsRepository def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: @@ -33,21 +29,6 @@ def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refuse monkeypatch.setattr(_hashlib, "new", new) -@pytest.mark.parametrize( - "general_settings, expected", - [ - ({"fips_mode": True}, True), - ({"fips_mode": " TRUE "}, True), - ({"fips_mode": False}, False), - ({"fips_mode": "false"}, False), - ({"fips_mode": None}, False), - ({}, False), - ], -) -def test_is_fips_mode(general_settings: Mapping[str, object], expected: bool) -> None: - assert is_fips_mode(general_settings) is expected - - @pytest.mark.parametrize( "fips_flag, md5_refused, enforces", [(1, True, True), (0, True, False), (1, False, False)], @@ -133,99 +114,3 @@ def test_enforce_fips_mode( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode(general_settings) - - -class _Database: - """Prisma's raw-query edge over a database whose only LiteLLM table besides LiteLLM_Config is the key table.""" - - def __init__(self, *, marked_fips: bool, holds_rows: bool, readable: bool = True, dropped_reads: int = 0) -> None: - self.marked_fips = marked_fips - self.holds_rows = holds_rows - self.readable = readable - self.dropped_reads = dropped_reads - - async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: - if self.dropped_reads: - self.dropped_reads -= 1 - raise httpx.ReadError("connection dropped") - if not self.readable: - raise RuntimeError('relation "LiteLLM_Config" does not exist') - if "information_schema.tables" in query: - return [{"table_name": "LiteLLM_VerificationToken"}] - if '"LiteLLM_VerificationToken"' in query: - return [{"marked_fips": self.marked_fips, "holds_rows": self.holds_rows}] - return [{"?column?": 1}] if self.marked_fips else [] - - async def execute_raw(self, query: str, *args: object) -> int: - self.marked_fips = True - return 1 - - -class _Prisma: - def __init__(self, database: _Database) -> None: - self.writer_db: Final = database - - async def attempt_db_reconnect( - self, *, reason: str, timeout_seconds: float | None = None, lock_timeout_seconds: float | None = None - ) -> bool: - return True - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "fips_mode, database, refusal, marked_after", - [ - (True, _Database(marked_fips=False, holds_rows=False), None, True), - (True, _Database(marked_fips=True, holds_rows=True), None, True), - (True, _Database(marked_fips=False, holds_rows=True), "already holds data written without it", False), - (True, _Database(marked_fips=False, holds_rows=False, readable=False), "could not be checked", False), - (False, _Database(marked_fips=True, holds_rows=True), "set up with general_settings.fips_mode: true", True), - (False, _Database(marked_fips=False, holds_rows=True), None, False), - (False, _Database(marked_fips=True, holds_rows=True, readable=False), None, True), - ( - True, - _Database(marked_fips=False, holds_rows=True, dropped_reads=1), - "already holds data written without it", - False, - ), - ( - False, - _Database(marked_fips=True, holds_rows=True, dropped_reads=1), - "set up with general_settings.fips_mode: true", - True, - ), - (False, _Database(marked_fips=True, holds_rows=True, dropped_reads=2), None, True), - ], - ids=[ - "fips-on-new-database", - "fips-on-fips-database", - "fips-on-used-database", - "fips-on-unreadable-database", - "fips-off-fips-database", - "fips-off-used-database", - "fips-off-unreadable-database", - "fips-on-used-database-after-a-dropped-connection", - "fips-off-fips-database-after-a-dropped-connection", - "fips-off-fips-database-dropped-twice", - ], -) -async def test_enforce_fips_database( - fips_mode: bool, database: _Database, refusal: str | None, marked_after: bool -) -> None: - with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): - await enforce_fips_database(FipsRepository(_Prisma(database)), fips_mode=fips_mode, database_is_configured=True) - - assert database.marked_fips is marked_after - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "fips_mode, database_is_configured, refusal", - [(True, True, "can't be reached"), (True, False, None), (False, True, None)], - ids=["fips-on-unreachable", "fips-on-no-database", "fips-off-unreachable"], -) -async def test_enforce_fips_database_without_a_connected_database( - fips_mode: bool, database_is_configured: bool, refusal: str | None -) -> None: - with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): - await enforce_fips_database(None, fips_mode=fips_mode, database_is_configured=database_is_configured) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index c46c1b336ae..8def4b0541a 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10,7 +10,7 @@ import subprocess import sys import time import types -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -1790,37 +1790,6 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database -class _DatabaseSetUpInFipsMode: - async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: - return [{"?column?": 1}] - - async def execute_raw(self, query: str, *args: object) -> int: - raise AssertionError("a refused boot must not write to the database") - - -class _PrismaClientOnAFipsDatabase: - def __init__(self) -> None: - self.writer_db: Final = _DatabaseSetUpInFipsMode() - - async def aclose(self) -> None: - pass - - -@pytest.mark.asyncio -async def test_proxy_startup_refuses_a_database_set_up_in_fips_mode_when_fips_mode_is_off( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from litellm.proxy.common_utils.fips import FipsModeError - from litellm.proxy.proxy_server import proxy_startup_event - - _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _PrismaClientOnAFipsDatabase()) - - with pytest.raises(FipsModeError, match=r"set up with general_settings\.fips_mode: true"): - async with proxy_startup_event(FastAPI()): - pass - - @pytest.mark.asyncio async def test_proxy_startup_reads_fips_mode_from_the_config_file( monkeypatch: pytest.MonkeyPatch, tmp_path: Path From 998befdf5573dd07dbe6186c60727442ba9f035c Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 2 Nov 2026 12:03:44 -0800 Subject: [PATCH 14/14] test(fips): pin how a fips_mode string is read through enforce_fips_mode --- tests/unit/proxy/common_utils/test_fips.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index e81577f79a4..a72772df492 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -82,9 +82,11 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: [ ({}, False, False, None), ({"fips_mode": False}, False, False, None), + ({"fips_mode": "false"}, False, False, None), ({"fips_mode": True}, True, True, None), ({"fips_mode": "true"}, True, True, None), ({"fips_mode": True}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"), + ({"fips_mode": " TRUE "}, False, True, "general_settings.fips_mode is on but OpenSSL does not enforce FIPS"), ({"fips_mode": True}, True, False, "cryptography library's OpenSSL has no working FIPS provider"), ({"fips_mode": "enforced"}, True, True, "general_settings.fips_mode='enforced' is not a boolean"), ({"fips_mode": 1}, True, True, "general_settings.fips_mode=1 is not a boolean"), @@ -93,9 +95,11 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: ids=[ "unset", "false", + "false-from-a-string", "fips", "fips-from-a-string", "openssl-not-fips", + "openssl-not-fips-from-a-padded-string", "cryptography-not-fips", "not-a-boolean", "a-number",