From db7c9aa5469d0e1e8f09cc959be7c6efa06e6eb8 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 10:31:54 -0700 Subject: [PATCH 1/8] feat(proxy): add LITELLM_FIPS_MODE startup gate With LITELLM_FIPS_MODE=true the proxy refuses to start when TLS verification is off, when Python's OpenSSL does not enforce FIPS, or when the startup password migration fails. A value that is not true/false also refuses. --- litellm/proxy/common_utils/fips.py | 53 ++++++++++++ litellm/proxy/proxy_server.py | 10 ++- .../configuration/test_fips_mode_boot.py | 62 ++++++++++++++ tests/unit/proxy/common_utils/test_fips.py | 85 +++++++++++++++++++ ...test_proxy_server_endpoints_and_startup.py | 39 +++++++++ 5 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/common_utils/fips.py create mode 100644 tests/integration/configuration/test_fips_mode_boot.py create mode 100644 tests/unit/proxy/common_utils/test_fips.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..3a8df10b9e5 --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,53 @@ +import os +from typing import Final + +from litellm.secret_managers.main import str_to_bool + +FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" + + +class FipsModeError(Exception): + pass + + +def is_fips_mode() -> bool: + return str_to_bool(os.getenv(FIPS_MODE_ENV_VAR)) is True + + +def openssl_enforces_fips() -> bool: + """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" + try: + import _hashlib + + if _hashlib.get_fips_mode() != 1: + return False + except (ImportError, AttributeError, ValueError): + return False + try: + _hashlib.new("md5", b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def enforce_fips_mode() -> None: + """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off + or OpenSSL does not enforce FIPS.""" + value: Final = os.getenv(FIPS_MODE_ENV_VAR) + if value and str_to_bool(value) is None: + raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") + if not is_fips_mode(): + return + + from litellm.llms.custom_httpx.http_handler import get_ssl_verify + + if get_ssl_verify() is False: + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY or " + "litellm_settings.ssl_verify. Remove the override, or point ssl_verify at a CA bundle." + ) + if not openssl_enforces_fips(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " + f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bcba76ca1b3..5aacc7ad1e6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -489,6 +489,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1523,6 +1524,8 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState if decoded_worker_config is not None: await initialize_from_worker_config(decoded_worker_config) + enforce_fips_mode() + enforce_master_key_boot_verdict( await with_stored_secrets_counted( master_key_boot_verdict( @@ -1565,9 +1568,14 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState result: Final = await migrate_passwords_to_scrypt_async(prisma_client) verbose_proxy_logger.info("Password migration: %s", result) except Exception as e: + if is_fips_mode(): + raise FipsModeError(f"Password migration failed, so plaintext passwords stay unhashed: {e}") from e verbose_proxy_logger.warning("Password migration skipped: %s", e) - asyncio.create_task(_run_pw_migration()) + if is_fips_mode(): + await _run_pw_migration() + else: + asyncio.create_task(_run_pw_migration()) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..85c7011876a --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,62 @@ +import _hashlib +import json +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway +from tests.integration._support.process import owned_proxy, refused_boot_log + + +def _config(directory: Path, upstream_url: str, *, ssl_verify: bool = True) -> Path: + config: Final = directory / "fips_boot.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "fips-model", + "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, + } + ], + "litellm_settings": {"ssl_verify": ssl_verify}, + } + ) + ) + return config + + +def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + log: Final = refused_boot_log( + gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=_config(tmp_path, gateway.upstream_url) + ) + assert "LITELLM_FIPS_MODE is on but OpenSSL does not enforce FIPS" in log, log + + +@pytest.mark.parametrize( + "overrides, ssl_verify", + [({"SSL_VERIFY": "false"}, True), ({}, False)], + ids=["ssl-verify-env", "litellm-settings"], +) +def test_fips_mode_refuses_to_start_with_tls_verification_disabled( + gateway: Gateway, tmp_path: Path, overrides: dict[str, str], ssl_verify: bool +) -> None: + config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=ssl_verify) + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", **overrides}, config=config) + assert "LITELLM_FIPS_MODE is on but TLS certificate verification is disabled" in log, log + + +def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log( + gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) + ) + assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log + + +def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: + config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) + with owned_proxy(gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config) as proxy: + assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..392db0d9cb4 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -0,0 +1,85 @@ +import _hashlib +from contextlib import nullcontext + +import pytest + +import litellm +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode, openssl_enforces_fips + + +def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new = _hashlib.new + + def new(name, data=b"", **kwargs): + if name == "md5" and md5_refused: + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, **kwargs) + + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) + monkeypatch.setattr(_hashlib, "new", new) + + +@pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) +def test_is_fips_mode(monkeypatch, value, expected): + if value is None: + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + else: + monkeypatch.setenv("LITELLM_FIPS_MODE", value) + assert is_fips_mode() is expected + + +@pytest.mark.parametrize( + "fips_flag, md5_refused, enforces", + [(1, True, True), (0, True, False), (1, False, False)], + ids=["fips", "fips-flag-off", "md5-allowed"], +) +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(monkeypatch, fips_flag, md5_refused, enforces): + _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) + assert openssl_enforces_fips() is enforces + + +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): + monkeypatch.delattr(_hashlib, "get_fips_mode") + assert openssl_enforces_fips() is False + + +def test_this_runners_openssl_does_not_enforce_fips(): + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + assert openssl_enforces_fips() is False + + +@pytest.mark.parametrize( + "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal", + [ + (None, "false", False, False, None), + ("", "false", False, False, None), + ("false", "false", False, False, None), + ("true", None, True, True, None), + ("true", "false", True, True, "TLS certificate verification is disabled"), + ("true", None, False, True, "TLS certificate verification is disabled"), + ("true", None, True, False, "OpenSSL does not enforce FIPS"), + ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + ], + ids=[ + "unset", + "empty", + "false", + "fips", + "ssl-verify-env-off", + "ssl-verify-setting-off", + "openssl-not-fips", + "not-a-boolean", + ], +) +def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal): + for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) + monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) + _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + enforce_fips_mode() diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 745488bf93f..12f0161ab0c 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1775,6 +1775,45 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + config_path, _ = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + config_path.write_text(yaml.dump({"litellm_settings": {"ssl_verify": False}})) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError, match="TLS certificate verification is disabled"): + async with proxy_startup_event(FastAPI()): + pass + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(monkeypatch, tmp_path): + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) + monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) + scrypt_refused = ValueError("[digital envelope routines] unsupported") + monkeypatch.setattr( + "litellm.proxy.proxy_server.migrate_passwords_to_scrypt_async", AsyncMock(side_effect=scrypt_refused) + ) + + with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: + async with proxy_startup_event(FastAPI()): + pass + + assert refused.value.__cause__ is scrypt_refused + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext From 75c6d07a8f87d7beb92e4304be8da24bf5046aa2 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 14:17:56 -0700 Subject: [PATCH 2/8] feat(proxy): refuse FIPS mode when cryptography's OpenSSL is not FIPS The stock cryptography wheel bundles its own non-FIPS OpenSSL, which serves JWT, Fernet and AES-GCM even when Python's OpenSSL enforces FIPS. --- litellm/proxy/common_utils/fips.py | 17 ++++++- tests/unit/proxy/common_utils/test_fips.py | 46 ++++++++++++++----- ...test_proxy_server_endpoints_and_startup.py | 1 + 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 3a8df10b9e5..f57f56d3248 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -30,9 +30,19 @@ def openssl_enforces_fips() -> bool: return False +def cryptography_enforces_fips() -> bool: + """The cryptography library's own OpenSSL runs with FIPS-only defaults.""" + try: + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + + return rust_openssl.is_fips_enabled() + except (ImportError, AttributeError): + return False + + def enforce_fips_mode() -> None: """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or OpenSSL does not enforce FIPS.""" + or Python's or the cryptography library's OpenSSL does not enforce FIPS.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") @@ -51,3 +61,8 @@ def enforce_fips_mode() -> None: f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." ) + if not cryptography_enforces_fips(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL does not enforce FIPS. Build it against " + f"the system OpenSSL (pip install --no-binary cryptography cryptography), or unset {FIPS_MODE_ENV_VAR}." + ) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 392db0d9cb4..98e95263dcb 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -2,9 +2,16 @@ import _hashlib from contextlib import nullcontext import pytest +from cryptography.hazmat.bindings._rust import openssl as rust_openssl import litellm -from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode, openssl_enforces_fips +from litellm.proxy.common_utils.fips import ( + FipsModeError, + cryptography_enforces_fips, + enforce_fips_mode, + is_fips_mode, + openssl_enforces_fips, +) def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: @@ -43,23 +50,36 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): assert openssl_enforces_fips() is False +@pytest.mark.parametrize("fips_enabled", [True, False]) +def test_cryptography_enforces_fips_reads_its_openssl_fips_flag(monkeypatch, fips_enabled): + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) + assert cryptography_enforces_fips() is fips_enabled + + +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): + monkeypatch.delattr(rust_openssl, "is_fips_enabled") + assert cryptography_enforces_fips() is False + + def test_this_runners_openssl_does_not_enforce_fips(): if _hashlib.get_fips_mode() == 1: pytest.skip("this runner's OpenSSL is in FIPS mode") assert openssl_enforces_fips() is False + assert cryptography_enforces_fips() is False @pytest.mark.parametrize( - "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal", + "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal", [ - (None, "false", False, False, None), - ("", "false", False, False, None), - ("false", "false", False, False, None), - ("true", None, True, True, None), - ("true", "false", True, True, "TLS certificate verification is disabled"), - ("true", None, False, True, "TLS certificate verification is disabled"), - ("true", None, True, False, "OpenSSL does not enforce FIPS"), - ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + (None, "false", False, False, False, None), + ("", "false", False, False, False, None), + ("false", "false", False, False, False, None), + ("true", None, True, True, True, None), + ("true", "false", True, True, True, "TLS certificate verification is disabled"), + ("true", None, False, True, True, "TLS certificate verification is disabled"), + ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), + ("true", None, True, True, False, "cryptography library's OpenSSL does not enforce FIPS"), + ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], ids=[ "unset", @@ -69,10 +89,13 @@ def test_this_runners_openssl_does_not_enforce_fips(): "ssl-verify-env-off", "ssl-verify-setting-off", "openssl-not-fips", + "cryptography-not-fips", "not-a-boolean", ], ) -def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, refusal): +def test_enforce_fips_mode( + monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal +): for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): if value is None: monkeypatch.delenv(name, raising=False) @@ -80,6 +103,7 @@ def test_enforce_fips_mode(monkeypatch, fips_mode, ssl_verify_env, ssl_verify_se monkeypatch.setenv(name, value) monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 12f0161ab0c..031c772dc44 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1801,6 +1801,7 @@ async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) + monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) scrypt_refused = ValueError("[digital envelope routines] unsupported") monkeypatch.setattr( From 23ebd158bf15861b102afd86e0edb090b54e450e Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 16:33:22 -0700 Subject: [PATCH 3/8] fix(proxy): refuse FIPS mode when cryptography's FIPS provider cannot hash The stock cryptography wheel reads the host's FIPS config and reports FIPS on, but its bundled OpenSSL cannot load the host's FIPS provider, so every operation fails. Require a working SHA-256 alongside the FIPS flag. --- litellm/proxy/common_utils/fips.py | 20 ++++++++++++++------ tests/unit/proxy/common_utils/test_fips.py | 19 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index f57f56d3248..671c023bbdb 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -31,13 +31,20 @@ def openssl_enforces_fips() -> bool: def cryptography_enforces_fips() -> bool: - """The cryptography library's own OpenSSL runs with FIPS-only defaults.""" + """The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256.""" try: + from cryptography.exceptions import InternalError, UnsupportedAlgorithm from cryptography.hazmat.bindings._rust import openssl as rust_openssl - - return rust_openssl.is_fips_enabled() - except (ImportError, AttributeError): + from cryptography.hazmat.primitives import hashes + except ImportError: return False + try: + if not rust_openssl.is_fips_enabled(): + return False + hashes.Hash(hashes.SHA256()).finalize() + except (AttributeError, InternalError, UnsupportedAlgorithm): + return False + return True def enforce_fips_mode() -> None: @@ -63,6 +70,7 @@ def enforce_fips_mode() -> None: ) if not cryptography_enforces_fips(): raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL does not enforce FIPS. Build it against " - f"the system OpenSSL (pip install --no-binary cryptography cryptography), or unset {FIPS_MODE_ENV_VAR}." + f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " + "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " + f"{FIPS_MODE_ENV_VAR}." ) diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 98e95263dcb..bd6834af536 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,8 +1,11 @@ import _hashlib from contextlib import nullcontext +from unittest.mock import Mock import pytest +from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes import litellm from litellm.proxy.common_utils.fips import ( @@ -50,10 +53,18 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): assert openssl_enforces_fips() is False -@pytest.mark.parametrize("fips_enabled", [True, False]) -def test_cryptography_enforces_fips_reads_its_openssl_fips_flag(monkeypatch, fips_enabled): +@pytest.mark.parametrize( + "fips_enabled, sha256_broken, enforces", + [(True, False, True), (False, False, False), (True, True, False)], + ids=["fips", "fips-flag-off", "sha256-broken"], +) +def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( + monkeypatch, fips_enabled, sha256_broken, enforces +): monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) - assert cryptography_enforces_fips() is fips_enabled + if sha256_broken: + monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) + assert cryptography_enforces_fips() is enforces def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): @@ -78,7 +89,7 @@ def test_this_runners_openssl_does_not_enforce_fips(): ("true", "false", True, True, True, "TLS certificate verification is disabled"), ("true", None, False, True, True, "TLS certificate verification is disabled"), ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), - ("true", None, True, True, False, "cryptography library's OpenSSL does not enforce FIPS"), + ("true", None, True, True, False, "cryptography library's OpenSSL has no working FIPS provider"), ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], ids=[ From 28c9a0d577894364c08ce8d9c440de1ec96c5f59 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 18:03:13 -0700 Subject: [PATCH 4/8] fix(proxy): refuse TLS verification off anywhere in FIPS mode A model's own ssl_verify: false reached the HTTP client after the startup check passed. get_ssl_verify now raises in FIPS mode whenever verification resolves to off, which also covers the startup check. --- litellm/llms/custom_httpx/http_handler.py | 10 ++++++++++ litellm/proxy/common_utils/fips.py | 6 +----- tests/unit/proxy/common_utils/test_fips.py | 8 ++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0b64885df45..ae9aa8f19d2 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -370,6 +370,16 @@ def get_ssl_verify( if ssl_cert_file and os.path.exists(ssl_cert_file): return ssl_cert_file + if ssl_verify is False: + from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode + + if is_fips_mode(): + raise FipsModeError( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " + "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a " + "CA bundle." + ) + return ssl_verify if ssl_verify is not None else True diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 671c023bbdb..2ffb17852eb 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -58,11 +58,7 @@ def enforce_fips_mode() -> None: from litellm.llms.custom_httpx.http_handler import get_ssl_verify - if get_ssl_verify() is False: - raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY or " - "litellm_settings.ssl_verify. Remove the override, or point ssl_verify at a CA bundle." - ) + get_ssl_verify() # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off if not openssl_enforces_fips(): raise FipsModeError( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index bd6834af536..e70fd2e3b46 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -8,6 +8,7 @@ from cryptography.hazmat.bindings._rust import openssl as rust_openssl from cryptography.hazmat.primitives import hashes import litellm +from litellm.llms.custom_httpx.http_handler import get_ssl_verify from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, @@ -118,3 +119,10 @@ def test_enforce_fips_mode( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() + + +@pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) +def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused): + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): + assert get_ssl_verify(ssl_verify=False) is False From 6103a8c60c16b2641d73eacd279ced26b6bf2f7e Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 18:33:09 -0700 Subject: [PATCH 5/8] refactor(proxy): return FIPS boot refusals as values and harden the FIPS tests --- litellm/llms/custom_httpx/http_handler.py | 25 ++++++---- litellm/proxy/common_utils/fips.py | 31 ++++++++---- .../configuration/test_fips_mode_boot.py | 4 +- tests/unit/proxy/common_utils/test_fips.py | 46 ++++++++++++------ ...test_proxy_server_endpoints_and_startup.py | 48 +++++++++++++++---- 5 files changed, 109 insertions(+), 45 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index ae9aa8f19d2..81cd624883b 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -336,6 +336,21 @@ def _create_ssl_context( def get_ssl_verify( ssl_verify: bool | str | None = None, +) -> bool | str: + """resolve_ssl_verify, refusing disabled TLS verification under LITELLM_FIPS_MODE.""" + resolved: Final = resolve_ssl_verify(ssl_verify) + if resolved is not False: + return resolved + + from litellm.proxy.common_utils.fips import TLS_VERIFICATION_DISABLED, FipsModeError, is_fips_mode + + if is_fips_mode(): + raise FipsModeError(TLS_VERIFICATION_DISABLED) + return resolved + + +def resolve_ssl_verify( + ssl_verify: bool | str | None = None, ) -> bool | str: """ Common utility to resolve the SSL verification setting. @@ -370,16 +385,6 @@ def get_ssl_verify( if ssl_cert_file and os.path.exists(ssl_cert_file): return ssl_cert_file - if ssl_verify is False: - from litellm.proxy.common_utils.fips import FIPS_MODE_ENV_VAR, FipsModeError, is_fips_mode - - if is_fips_mode(): - raise FipsModeError( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " - "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a " - "CA bundle." - ) - return ssl_verify if ssl_verify is not None else True diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 2ffb17852eb..2cbe154d2a1 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -4,6 +4,10 @@ from typing import Final from litellm.secret_managers.main import str_to_bool FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +TLS_VERIFICATION_DISABLED: Final = ( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " + "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a CA bundle." +) class FipsModeError(Exception): @@ -47,26 +51,35 @@ def cryptography_enforces_fips() -> bool: return True -def enforce_fips_mode() -> None: - """Refuse to start when LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or Python's or the cryptography library's OpenSSL does not enforce FIPS.""" +def fips_mode_refusal() -> str | None: + """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off + or Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: - raise FipsModeError(f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it.") + return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it." if not is_fips_mode(): - return + return None - from litellm.llms.custom_httpx.http_handler import get_ssl_verify + from litellm.llms.custom_httpx.http_handler import resolve_ssl_verify - get_ssl_verify() # raises FipsModeError when SSL_VERIFY or litellm_settings.ssl_verify turns verification off + if resolve_ssl_verify() is False: + return TLS_VERIFICATION_DISABLED if not openssl_enforces_fips(): - raise FipsModeError( + return ( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." ) if not cryptography_enforces_fips(): - raise FipsModeError( + return ( f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " f"{FIPS_MODE_ENV_VAR}." ) + return None + + +def enforce_fips_mode() -> None: + """Raise FipsModeError with the fips_mode_refusal reason, if there is one.""" + refusal: Final = fips_mode_refusal() + if refusal is not None: + raise FipsModeError(refusal) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index 85c7011876a..0a20d14fb19 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -58,5 +58,7 @@ def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Ga def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) - with owned_proxy(gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config) as proxy: + with owned_proxy( + gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config, remove_environment=("LITELLM_FIPS_MODE",) + ) as proxy: assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index e70fd2e3b46..596d45f88d1 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,5 +1,6 @@ import _hashlib from contextlib import nullcontext +from typing import Final from unittest.mock import Mock import pytest @@ -18,20 +19,20 @@ from litellm.proxy.common_utils.fips import ( ) -def _fake_openssl(monkeypatch, *, fips_flag: int, md5_refused: bool) -> None: - real_new = _hashlib.new +def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new: Final = _hashlib.new - def new(name, data=b"", **kwargs): + def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: if name == "md5" and md5_refused: raise _hashlib.UnsupportedDigestmodError("unsupported hash type") - return real_new(name, data, **kwargs) + return real_new(name, data, usedforsecurity=usedforsecurity) monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) monkeypatch.setattr(_hashlib, "new", new) @pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) -def test_is_fips_mode(monkeypatch, value, expected): +def test_is_fips_mode(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool) -> None: if value is None: monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) else: @@ -44,12 +45,14 @@ def test_is_fips_mode(monkeypatch, value, expected): [(1, True, True), (0, True, False), (1, False, False)], ids=["fips", "fips-flag-off", "md5-allowed"], ) -def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5(monkeypatch, fips_flag, md5_refused, enforces): +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5( + monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool +) -> None: _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) assert openssl_enforces_fips() is enforces -def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(_hashlib, "get_fips_mode") assert openssl_enforces_fips() is False @@ -60,23 +63,28 @@ def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch): ids=["fips", "fips-flag-off", "sha256-broken"], ) def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( - monkeypatch, fips_enabled, sha256_broken, enforces -): + monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool +) -> None: monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) if sha256_broken: monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) assert cryptography_enforces_fips() is enforces -def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch): +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delattr(rust_openssl, "is_fips_enabled") assert cryptography_enforces_fips() is False -def test_this_runners_openssl_does_not_enforce_fips(): +def test_this_runners_python_openssl_does_not_enforce_fips() -> None: if _hashlib.get_fips_mode() == 1: - pytest.skip("this runner's OpenSSL is in FIPS mode") + pytest.skip("this runner's Python OpenSSL is in FIPS mode") assert openssl_enforces_fips() is False + + +def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: + if rust_openssl.is_fips_enabled(): + pytest.skip("this runner's cryptography OpenSSL is in FIPS mode") assert cryptography_enforces_fips() is False @@ -106,8 +114,14 @@ def test_this_runners_openssl_does_not_enforce_fips(): ], ) def test_enforce_fips_mode( - monkeypatch, fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal -): + monkeypatch: pytest.MonkeyPatch, + fips_mode: str | None, + ssl_verify_env: str | None, + ssl_verify_setting: bool, + openssl_fips: bool, + cryptography_fips: bool, + refusal: str | None, +) -> None: for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): if value is None: monkeypatch.delenv(name, raising=False) @@ -122,7 +136,9 @@ def test_enforce_fips_mode( @pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) -def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode(monkeypatch, fips_mode, refused): +def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, refused: bool +) -> None: monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): assert get_ssl_verify(ssl_verify=False) is False diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 031c772dc44..4bdadbba02e 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1776,7 +1776,9 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.proxy_server import proxy_startup_event @@ -1791,22 +1793,48 @@ async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_veri pass +class _UsersWithOnePlaintextPassword: + async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: + return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) + + async def update(self, where: object, data: object) -> None: + raise AssertionError("a refused boot must not write to the database") + + +class _DatabaseWithOnePlaintextPassword: + def __init__(self) -> None: + self.db: Final = types.SimpleNamespace(litellm_usertable=_UsersWithOnePlaintextPassword()) + + @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails(monkeypatch, tmp_path): +async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import _hashlib + import hashlib + + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + from litellm.proxy.common_utils.fips import FipsModeError - from litellm.proxy.proxy_server import ProxyStartupEvent, proxy_startup_event + from litellm.proxy.proxy_server import proxy_startup_event + + real_new: Final = _hashlib.new + + def new_refusing_md5(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: + if name == "md5": + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, usedforsecurity=usedforsecurity) _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) monkeypatch.setattr(litellm, "ssl_verify", True) monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - monkeypatch.setattr("litellm.proxy.common_utils.fips.openssl_enforces_fips", lambda: True) - monkeypatch.setattr("litellm.proxy.common_utils.fips.cryptography_enforces_fips", lambda: True) - monkeypatch.setattr(ProxyStartupEvent, "setup_prisma_client", AsyncMock(return_value=MockPrisma())) - scrypt_refused = ValueError("[digital envelope routines] unsupported") - monkeypatch.setattr( - "litellm.proxy.proxy_server.migrate_passwords_to_scrypt_async", AsyncMock(side_effect=scrypt_refused) - ) + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr(_hashlib, "new", new_refusing_md5) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + scrypt_refused: Final = ValueError("[digital envelope routines] unsupported") + monkeypatch.setattr(hashlib, "scrypt", mock.Mock(side_effect=scrypt_refused)) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _DatabaseWithOnePlaintextPassword()) with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: async with proxy_startup_event(FastAPI()): From bd24880ee5b46f6229b44b9a8d9a79c642630171 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 30 Oct 2026 18:03:09 -0700 Subject: [PATCH 6/8] fix(proxy): stop refusing TLS verification off in FIPS mode Skipping certificate verification is a security choice, not a FIPS one: the TLS connection still only uses algorithms the FIPS provider allows. --- litellm/llms/custom_httpx/http_handler.py | 15 ------- litellm/proxy/common_utils/fips.py | 13 +------ .../configuration/test_fips_mode_boot.py | 26 +------------ tests/unit/proxy/common_utils/test_fips.py | 39 ++++++------------- ...test_proxy_server_endpoints_and_startup.py | 20 ---------- 5 files changed, 16 insertions(+), 97 deletions(-) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 81cd624883b..0b64885df45 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -336,21 +336,6 @@ def _create_ssl_context( def get_ssl_verify( ssl_verify: bool | str | None = None, -) -> bool | str: - """resolve_ssl_verify, refusing disabled TLS verification under LITELLM_FIPS_MODE.""" - resolved: Final = resolve_ssl_verify(ssl_verify) - if resolved is not False: - return resolved - - from litellm.proxy.common_utils.fips import TLS_VERIFICATION_DISABLED, FipsModeError, is_fips_mode - - if is_fips_mode(): - raise FipsModeError(TLS_VERIFICATION_DISABLED) - return resolved - - -def resolve_ssl_verify( - ssl_verify: bool | str | None = None, ) -> bool | str: """ Common utility to resolve the SSL verification setting. diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 2cbe154d2a1..155696ea70f 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -4,10 +4,6 @@ from typing import Final from litellm.secret_managers.main import str_to_bool FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" -TLS_VERIFICATION_DISABLED: Final = ( - f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " - "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a CA bundle." -) class FipsModeError(Exception): @@ -52,18 +48,13 @@ def cryptography_enforces_fips() -> bool: def fips_mode_refusal() -> str | None: - """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off - or Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" + """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but Python's or the + cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" value: Final = os.getenv(FIPS_MODE_ENV_VAR) if value and str_to_bool(value) is None: return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it." if not is_fips_mode(): return None - - from litellm.llms.custom_httpx.http_handler import resolve_ssl_verify - - if resolve_ssl_verify() is False: - return TLS_VERIFICATION_DISABLED if not openssl_enforces_fips(): return ( f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index 0a20d14fb19..e2d7975b252 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -6,10 +6,10 @@ from typing import Final import pytest from tests.integration._support.client import Gateway -from tests.integration._support.process import owned_proxy, refused_boot_log +from tests.integration._support.process import refused_boot_log -def _config(directory: Path, upstream_url: str, *, ssl_verify: bool = True) -> Path: +def _config(directory: Path, upstream_url: str) -> Path: config: Final = directory / "fips_boot.yaml" config.write_text( json.dumps( @@ -20,7 +20,6 @@ def _config(directory: Path, upstream_url: str, *, ssl_verify: bool = True) -> P "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, } ], - "litellm_settings": {"ssl_verify": ssl_verify}, } ) ) @@ -36,29 +35,8 @@ def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: assert "LITELLM_FIPS_MODE is on but OpenSSL does not enforce FIPS" in log, log -@pytest.mark.parametrize( - "overrides, ssl_verify", - [({"SSL_VERIFY": "false"}, True), ({}, False)], - ids=["ssl-verify-env", "litellm-settings"], -) -def test_fips_mode_refuses_to_start_with_tls_verification_disabled( - gateway: Gateway, tmp_path: Path, overrides: dict[str, str], ssl_verify: bool -) -> None: - config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=ssl_verify) - log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", **overrides}, config=config) - assert "LITELLM_FIPS_MODE is on but TLS certificate verification is disabled" in log, log - - def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: log: Final = refused_boot_log( gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) ) assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log - - -def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: - config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) - with owned_proxy( - gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config, remove_environment=("LITELLM_FIPS_MODE",) - ) as proxy: - assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index 596d45f88d1..a2afea25306 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -8,8 +8,6 @@ from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._rust import openssl as rust_openssl from cryptography.hazmat.primitives import hashes -import litellm -from litellm.llms.custom_httpx.http_handler import get_ssl_verify from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, @@ -89,25 +87,23 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: @pytest.mark.parametrize( - "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal", + "fips_mode, ssl_verify, openssl_fips, cryptography_fips, refusal", [ - (None, "false", False, False, False, None), - ("", "false", False, False, False, None), - ("false", "false", False, False, False, None), - ("true", None, True, True, True, None), - ("true", "false", True, True, True, "TLS certificate verification is disabled"), - ("true", None, False, True, True, "TLS certificate verification is disabled"), - ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), - ("true", None, True, True, False, "cryptography library's OpenSSL has no working FIPS provider"), - ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + (None, None, False, False, None), + ("", None, False, False, None), + ("false", None, False, False, None), + ("true", None, True, True, None), + ("true", "false", True, True, None), + ("true", None, False, True, "is on but OpenSSL does not enforce FIPS"), + ("true", None, True, False, "cryptography library's OpenSSL has no working FIPS provider"), + ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], ids=[ "unset", "empty", "false", "fips", - "ssl-verify-env-off", - "ssl-verify-setting-off", + "fips-with-tls-verification-off", "openssl-not-fips", "cryptography-not-fips", "not-a-boolean", @@ -116,29 +112,18 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: def test_enforce_fips_mode( monkeypatch: pytest.MonkeyPatch, fips_mode: str | None, - ssl_verify_env: str | None, - ssl_verify_setting: bool, + ssl_verify: str | None, openssl_fips: bool, cryptography_fips: bool, refusal: str | None, ) -> None: - for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): + for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify)): if value is None: monkeypatch.delenv(name, raising=False) else: monkeypatch.setenv(name, value) - monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() - - -@pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) -def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode( - monkeypatch: pytest.MonkeyPatch, fips_mode: str, refused: bool -) -> None: - monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) - with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): - assert get_ssl_verify(ssl_verify=False) is False diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 4bdadbba02e..ea238f74076 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1775,24 +1775,6 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database -@pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from litellm.proxy.common_utils.fips import FipsModeError - from litellm.proxy.proxy_server import proxy_startup_event - - config_path, _ = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) - config_path.write_text(yaml.dump({"litellm_settings": {"ssl_verify": False}})) - monkeypatch.setattr(litellm, "ssl_verify", True) - monkeypatch.delenv("SSL_VERIFY", raising=False) - monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - - with pytest.raises(FipsModeError, match="TLS certificate verification is disabled"): - async with proxy_startup_event(FastAPI()): - pass - - class _UsersWithOnePlaintextPassword: async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) @@ -1826,8 +1808,6 @@ async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails return real_new(name, data, usedforsecurity=usedforsecurity) _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) - monkeypatch.setattr(litellm, "ssl_verify", True) - monkeypatch.delenv("SSL_VERIFY", raising=False) monkeypatch.setenv("LITELLM_FIPS_MODE", "true") monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) monkeypatch.setattr(_hashlib, "new", new_refusing_md5) From f770015279d88b5f9ca32e4f75b9c0c79a8fe487 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 30 Oct 2026 19:33:15 -0700 Subject: [PATCH 7/8] feat(fips): refuse a database used without FIPS mode, drop the password migration change FIPS mode marks a new database with a fips_mode row in LiteLLM_Config and refuses one that already holds LiteLLM rows without it. Without FIPS mode, the proxy refuses a database carrying the marker. --- litellm/proxy/common_utils/fips.py | 49 ++++++++ litellm/proxy/proxy_server.py | 15 +-- litellm/repositories/fips_repository.py | 65 ++++++++++ .../configuration/test_fips_mode_boot.py | 25 ++++ .../database/test_fips_database.py | 93 ++++++++++++++ tests/unit/proxy/common_utils/test_fips.py | 116 ++++++++++++++---- ...test_proxy_server_endpoints_and_startup.py | 41 ++----- 7 files changed, 342 insertions(+), 62 deletions(-) create mode 100644 litellm/repositories/fips_repository.py create mode 100644 tests/integration/database/test_fips_database.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 155696ea70f..8c70b7d7825 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -1,6 +1,8 @@ import os from typing import Final +from litellm._logging import verbose_proxy_logger +from litellm.repositories.fips_repository import FIPS_MARKER, FipsRepository from litellm.secret_managers.main import str_to_bool FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" @@ -74,3 +76,50 @@ def enforce_fips_mode() -> None: refusal: Final = fips_mode_refusal() if refusal is not None: raise FipsModeError(refusal) + + +async def _is_marked_fips(database: FipsRepository) -> bool: + try: + return await database.is_marked_fips() + except Exception as error: # noqa: BLE001 # without FIPS mode, an unreadable database starts as it did before + verbose_proxy_logger.warning("Could not check whether the database was set up in FIPS mode: %s", error) + return False + + +async def fips_database_refusal(database: FipsRepository | None, *, database_is_configured: bool) -> str | None: + """Why the proxy must not start on its database: LITELLM_FIPS_MODE is off on a database set up in FIPS mode, or + on over a database that can't be checked or already holds rows written without it. None when it may start.""" + if not is_fips_mode(): + if database is not None and await _is_marked_fips(database): + return ( + f"This database was set up with {FIPS_MODE_ENV_VAR}=true, so the proxy only starts on it in FIPS mode. " + f"To stop using FIPS mode on it for good, run " + f"DELETE FROM \"LiteLLM_Config\" WHERE param_name = '{FIPS_MARKER}'." + ) + return None + if database is None: + return ( + f"{FIPS_MODE_ENV_VAR} is on but the database can't be reached to check it." + if database_is_configured + else None + ) + try: + state: Final = await database.state() + except Exception as error: # noqa: BLE001 # a value, so FIPS mode refuses a database it can't check + return f"{FIPS_MODE_ENV_VAR} is on but the database could not be checked: {error}" + if state.holds_rows and not state.marked_fips: + return ( + f"{FIPS_MODE_ENV_VAR} is on but this database already holds data written without it. " + "FIPS mode only starts on a new, empty database." + ) + return None + + +async def enforce_fips_database(database: FipsRepository | None, *, database_is_configured: bool) -> None: + """Raise FipsModeError with the fips_database_refusal reason, if there is one, then mark the database as set up + in FIPS mode when LITELLM_FIPS_MODE is on.""" + refusal: Final = await fips_database_refusal(database, database_is_configured=database_is_configured) + if refusal is not None: + raise FipsModeError(refusal) + if database is not None and is_fips_mode(): + await database.mark_fips() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 974eba6ca9d..57e36b0b62e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -499,7 +499,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id -from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode +from litellm.proxy.common_utils.fips import enforce_fips_database, enforce_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -992,6 +992,7 @@ from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.credentials_repository import CredentialsRepository +from litellm.repositories.fips_repository import FipsRepository from litellm.repositories.prisma_protocols import TableActions from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router import ( @@ -1640,6 +1641,11 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener database: Final = existing_database or await _own_database(resources, prisma_client) + await enforce_fips_database( + None if prisma_client is None else FipsRepository(prisma_client), + database_is_configured=get_secret("DATABASE_URL", None) is not None, + ) + await migrate_if_requested( environ=os.environ, master_key=master_key, @@ -1656,14 +1662,9 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener result: Final = await migrate_passwords_to_scrypt_async(startup_prisma) verbose_proxy_logger.info("Password migration: %s", result) except Exception as e: - if is_fips_mode(): - raise FipsModeError(f"Password migration failed, so plaintext passwords stay unhashed: {e}") from e verbose_proxy_logger.warning("Password migration skipped: %s", e) - if is_fips_mode(): - await _run_pw_migration() - else: - await resources.enter_async_context(database_runtime.background_task(_run_pw_migration())) + await resources.enter_async_context(database_runtime.background_task(_run_pw_migration())) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/litellm/repositories/fips_repository.py b/litellm/repositories/fips_repository.py new file mode 100644 index 00000000000..23a756e44bf --- /dev/null +++ b/litellm/repositories/fips_repository.py @@ -0,0 +1,65 @@ +"""FIPS repository for the LiteLLM_Config row that marks a database as set up in FIPS mode.""" + +from dataclasses import dataclass +from typing import Final, Protocol, cast # noqa: TID251 # Prisma delegates query_raw and execute_raw dynamically + +from litellm.proxy.db.create_views import SupportsRawQueries + +FIPS_MARKER: Final = "fips_mode" +_IS_MARKED: Final = 'SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1' +_OTHER_TABLES: Final = ( + "SELECT table_name FROM information_schema.tables WHERE table_schema = ANY (current_schemas(false)) " + "AND table_type = 'BASE TABLE' AND table_name LIKE 'LiteLLM\\_%' AND table_name <> 'LiteLLM_Config'" +) +_MARK: Final = ( + "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ($1, 'true'::jsonb) " + "ON CONFLICT (param_name) DO NOTHING" +) + + +def _quoted(identifier: str) -> str: + return '"' + identifier.replace('"', '""') + '"' + + +class _WithWriter(Protocol): + @property + def writer_db(self) -> object: ... + + +@dataclass(frozen=True, slots=True) +class FipsDatabaseState: + marked_fips: bool + holds_rows: bool + + +class FipsRepository: + """Reads and writes go to the writer, since a replica may not have the marker yet.""" + + def __init__(self, prisma_client: _WithWriter) -> None: + self._prisma_client: Final = prisma_client + + @property + def _writer(self) -> SupportsRawQueries: + writer: Final = self._prisma_client.writer_db + return cast(SupportsRawQueries, writer) # cast-ok: Prisma delegates database methods dynamically + + async def is_marked_fips(self) -> bool: + return bool(await self._writer.query_raw(_IS_MARKED, FIPS_MARKER)) + + async def state(self) -> FipsDatabaseState: + """The marker and whether any other LiteLLM row exists, read in one statement so they agree.""" + writer: Final = self._writer + tables: Final = tuple(str(row["table_name"]) for row in await writer.query_raw(_OTHER_TABLES)) + any_row_in_other_tables: Final = "".join(f" OR EXISTS (SELECT 1 FROM {_quoted(table)})" for table in tables) + row: Final = ( + await writer.query_raw( + 'SELECT EXISTS (SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1) AS marked_fips, ' + f'(EXISTS (SELECT 1 FROM "LiteLLM_Config" WHERE param_name <> $1){any_row_in_other_tables}) ' + "AS holds_rows", + FIPS_MARKER, + ) + )[0] + return FipsDatabaseState(marked_fips=row["marked_fips"] is True, holds_rows=row["holds_rows"] is True) + + async def mark_fips(self) -> None: + await self._writer.execute_raw(_MARK, FIPS_MARKER) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index e2d7975b252..11e1f8fc088 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -6,6 +6,7 @@ from typing import Final import pytest from tests.integration._support.client import Gateway +from tests.integration._support.database import scratch_database, write_rows from tests.integration._support.process import refused_boot_log @@ -40,3 +41,27 @@ def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Ga gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) ) assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log + + +def test_proxy_refuses_to_start_without_fips_mode_on_a_database_set_up_in_fips_mode( + gateway: Gateway, tmp_path: Path +) -> None: + with scratch_database() as database_url: + write_rows( + 'CREATE TABLE "LiteLLM_Config" (param_name TEXT NOT NULL PRIMARY KEY, param_value JSONB, ' + "last_run_at TIMESTAMP(3), reload_revision BIGINT NOT NULL DEFAULT 0)", + (), + database_url=database_url, + ) + write_rows( + "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('fips_mode', 'true')", + (), + database_url=database_url, + ) + log: Final = refused_boot_log( + gateway, + tmp_path, + {"LITELLM_FIPS_MODE": "false", "DATABASE_URL": database_url}, + config=_config(tmp_path, gateway.upstream_url), + ) + assert "This database was set up with LITELLM_FIPS_MODE=true" in log, log diff --git a/tests/integration/database/test_fips_database.py b/tests/integration/database/test_fips_database.py new file mode 100644 index 00000000000..21f86308fe2 --- /dev/null +++ b/tests/integration/database/test_fips_database.py @@ -0,0 +1,93 @@ +import asyncio +from contextlib import nullcontext +from typing import Final, LiteralString + +import pytest + +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_database +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.fips_repository import FipsRepository +from tests.integration._support.database import read_rows, scratch_database, write_rows + +_CONFIG_TABLE: Final = ( + 'CREATE TABLE "LiteLLM_Config" (param_name TEXT PRIMARY KEY, param_value JSONB, last_run_at TIMESTAMP(3), ' + "reload_revision BIGINT NOT NULL DEFAULT 0)" +) +_KEY_TABLE: Final = 'CREATE TABLE "LiteLLM_VerificationToken" (token TEXT PRIMARY KEY)' +_MARKER: Final = "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('fips_mode', 'true')" +_KEY: Final = "INSERT INTO \"LiteLLM_VerificationToken\" (token) VALUES ('hashed-key')" +_SETTING: Final = "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('general_settings', '{}')" + + +def _marker_rows(database_url: str) -> int: + return len( + read_rows('SELECT 1 FROM "LiteLLM_Config" WHERE param_name = %s', ("fips_mode",), database_url=database_url) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, setup, refusal, marked_after", + [ + ("true", (_CONFIG_TABLE, _KEY_TABLE), None, 1), + ("true", (_CONFIG_TABLE, _KEY_TABLE, _MARKER, _KEY), None, 1), + ("true", (_CONFIG_TABLE, _KEY_TABLE, _KEY), "already holds data written without it", 0), + ("true", (_CONFIG_TABLE, _KEY_TABLE, _SETTING), "already holds data written without it", 0), + ("true", (), "could not be checked", None), + ("false", (_CONFIG_TABLE, _KEY_TABLE, _MARKER), "set up with LITELLM_FIPS_MODE=true", 1), + ("false", (_CONFIG_TABLE, _KEY_TABLE, _KEY), None, 0), + ("false", (), None, None), + ], + ids=[ + "fips-on-new-database", + "fips-on-fips-database", + "fips-on-database-with-a-key", + "fips-on-database-with-a-setting", + "fips-on-database-without-tables", + "fips-off-fips-database", + "fips-off-database-with-a-key", + "fips-off-database-without-tables", + ], +) +async def test_fips_database_check( + monkeypatch: pytest.MonkeyPatch, + fips_mode: str, + setup: tuple[LiteralString, ...], + refusal: str | None, + marked_after: int | None, +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + with scratch_database() as database_url: + for statement in setup: + write_rows(statement, (), database_url=database_url) + monkeypatch.setenv("DATABASE_URL", database_url) + client: Final = PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(FipsRepository(client), database_is_configured=True) + finally: + await client.disconnect() + if marked_after is not None: + assert _marker_rows(database_url) == marked_after + + +@pytest.mark.asyncio +async def test_pods_starting_together_in_fips_mode_on_a_new_database_both_start( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + with scratch_database() as database_url: + for statement in (_CONFIG_TABLE, _KEY_TABLE): + write_rows(statement, (), database_url=database_url) + monkeypatch.setenv("DATABASE_URL", database_url) + pods: Final = tuple(PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) for _ in range(4)) + await asyncio.gather(*(pod.connect() for pod in pods)) + try: + await asyncio.gather( + *(enforce_fips_database(FipsRepository(pod), database_is_configured=True) for pod in pods) + ) + finally: + await asyncio.gather(*(pod.disconnect() for pod in pods)) + assert _marker_rows(database_url) == 1 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index a2afea25306..a48ba3138e2 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,4 +1,5 @@ import _hashlib +from collections.abc import Mapping, Sequence from contextlib import nullcontext from typing import Final from unittest.mock import Mock @@ -11,10 +12,12 @@ from cryptography.hazmat.primitives import hashes from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, + enforce_fips_database, enforce_fips_mode, is_fips_mode, openssl_enforces_fips, ) +from litellm.repositories.fips_repository import FipsRepository def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: @@ -87,43 +90,106 @@ def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: @pytest.mark.parametrize( - "fips_mode, ssl_verify, openssl_fips, cryptography_fips, refusal", + "fips_mode, openssl_fips, cryptography_fips, refusal", [ - (None, None, False, False, None), - ("", None, False, False, None), - ("false", None, False, False, None), - ("true", None, True, True, None), - ("true", "false", True, True, None), - ("true", None, False, True, "is on but OpenSSL does not enforce FIPS"), - ("true", None, True, False, "cryptography library's OpenSSL has no working FIPS provider"), - ("enforced", None, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), - ], - ids=[ - "unset", - "empty", - "false", - "fips", - "fips-with-tls-verification-off", - "openssl-not-fips", - "cryptography-not-fips", - "not-a-boolean", + (None, False, False, None), + ("", False, False, None), + ("false", False, False, None), + ("true", True, True, None), + ("true", False, True, "is on but OpenSSL does not enforce FIPS"), + ("true", True, False, "cryptography library's OpenSSL has no working FIPS provider"), + ("enforced", True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), ], + ids=["unset", "empty", "false", "fips", "openssl-not-fips", "cryptography-not-fips", "not-a-boolean"], ) def test_enforce_fips_mode( monkeypatch: pytest.MonkeyPatch, fips_mode: str | None, - ssl_verify: str | None, openssl_fips: bool, cryptography_fips: bool, refusal: str | None, ) -> None: - for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify)): - if value is None: - monkeypatch.delenv(name, raising=False) - else: - monkeypatch.setenv(name, value) + if fips_mode is None: + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + else: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode() + + +class _Database: + """Prisma's raw-query edge over a database whose only LiteLLM table besides LiteLLM_Config is the key table.""" + + def __init__(self, *, marked_fips: bool, holds_rows: bool, readable: bool = True) -> None: + self.marked_fips = marked_fips + self.holds_rows = holds_rows + self.readable = readable + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + if not self.readable: + raise RuntimeError('relation "LiteLLM_Config" does not exist') + if "information_schema.tables" in query: + return [{"table_name": "LiteLLM_VerificationToken"}] + if '"LiteLLM_VerificationToken"' in query: + return [{"marked_fips": self.marked_fips, "holds_rows": self.holds_rows}] + return [{"?column?": 1}] if self.marked_fips else [] + + async def execute_raw(self, query: str, *args: object) -> int: + self.marked_fips = True + return 1 + + +class _Prisma: + def __init__(self, database: _Database) -> None: + self.writer_db: Final = database + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, database, refusal, marked_after", + [ + ("true", _Database(marked_fips=False, holds_rows=False), None, True), + ("true", _Database(marked_fips=True, holds_rows=True), None, True), + ("true", _Database(marked_fips=False, holds_rows=True), "already holds data written without it", False), + ("true", _Database(marked_fips=False, holds_rows=False, readable=False), "could not be checked", False), + ("false", _Database(marked_fips=True, holds_rows=True), "set up with LITELLM_FIPS_MODE=true", True), + ("false", _Database(marked_fips=False, holds_rows=True), None, False), + ("false", _Database(marked_fips=True, holds_rows=True, readable=False), None, True), + ], + ids=[ + "fips-on-new-database", + "fips-on-fips-database", + "fips-on-used-database", + "fips-on-unreadable-database", + "fips-off-fips-database", + "fips-off-used-database", + "fips-off-unreadable-database", + ], +) +async def test_enforce_fips_database( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, database: _Database, refusal: str | None, marked_after: bool +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(FipsRepository(_Prisma(database)), database_is_configured=True) + + assert database.marked_fips is marked_after + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, database_is_configured, refusal", + [("true", True, "can't be reached"), ("true", False, None), ("false", True, None)], + ids=["fips-on-unreachable", "fips-on-no-database", "fips-off-unreachable"], +) +async def test_enforce_fips_database_without_a_connected_database( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, database_is_configured: bool, refusal: str | None +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(None, database_is_configured=database_is_configured) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 6ae5c47bfbc..18574e08a3e 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10,7 +10,7 @@ import subprocess import sys import time import types -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -1790,56 +1790,37 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database -class _UsersWithOnePlaintextPassword: - async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: - return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) +class _DatabaseSetUpInFipsMode: + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + return [{"?column?": 1}] - async def update(self, where: object, data: object) -> None: + async def execute_raw(self, query: str, *args: object) -> int: raise AssertionError("a refused boot must not write to the database") -class _DatabaseWithOnePlaintextPassword: +class _PrismaClientOnAFipsDatabase: def __init__(self) -> None: - self.db: Final = types.SimpleNamespace(litellm_usertable=_UsersWithOnePlaintextPassword()) + self.writer_db: Final = _DatabaseSetUpInFipsMode() async def aclose(self) -> None: pass @pytest.mark.asyncio -async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails( +async def test_proxy_startup_refuses_a_database_set_up_in_fips_mode_when_fips_mode_is_off( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - import _hashlib - import hashlib - - from cryptography.hazmat.bindings._rust import openssl as rust_openssl - from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.proxy_server import proxy_startup_event - real_new: Final = _hashlib.new - - def new_refusing_md5(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: - if name == "md5": - raise _hashlib.UnsupportedDigestmodError("unsupported hash type") - return real_new(name, data, usedforsecurity=usedforsecurity) - _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) - monkeypatch.setenv("LITELLM_FIPS_MODE", "true") - monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) - monkeypatch.setattr(_hashlib, "new", new_refusing_md5) - monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) - scrypt_refused: Final = ValueError("[digital envelope routines] unsupported") - monkeypatch.setattr(hashlib, "scrypt", mock.Mock(side_effect=scrypt_refused)) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _DatabaseWithOnePlaintextPassword()) + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _PrismaClientOnAFipsDatabase()) - with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: + with pytest.raises(FipsModeError, match="set up with LITELLM_FIPS_MODE=true"): async with proxy_startup_event(FastAPI()): pass - assert refused.value.__cause__ is scrypt_refused - class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): From ba6f2b840d4d1b529e937edb48a24b06d391f690 Mon Sep 17 00:00:00 2001 From: yucheng Date: Fri, 30 Oct 2026 19:59:19 -0700 Subject: [PATCH 8/8] test(fips): close the database test clients with aclose so the engine stops --- tests/integration/database/test_fips_database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/database/test_fips_database.py b/tests/integration/database/test_fips_database.py index 21f86308fe2..54238bc6876 100644 --- a/tests/integration/database/test_fips_database.py +++ b/tests/integration/database/test_fips_database.py @@ -68,7 +68,7 @@ async def test_fips_database_check( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): await enforce_fips_database(FipsRepository(client), database_is_configured=True) finally: - await client.disconnect() + await client.aclose() if marked_after is not None: assert _marker_rows(database_url) == marked_after @@ -89,5 +89,5 @@ async def test_pods_starting_together_in_fips_mode_on_a_new_database_both_start( *(enforce_fips_database(FipsRepository(pod), database_is_configured=True) for pod in pods) ) finally: - await asyncio.gather(*(pod.disconnect() for pod in pods)) + await asyncio.gather(*(pod.aclose() for pod in pods)) assert _marker_rows(database_url) == 1