diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 0b64885df45..81cd624883b 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -336,6 +336,21 @@ def _create_ssl_context( def get_ssl_verify( ssl_verify: bool | str | None = None, +) -> bool | str: + """resolve_ssl_verify, refusing disabled TLS verification under LITELLM_FIPS_MODE.""" + resolved: Final = resolve_ssl_verify(ssl_verify) + if resolved is not False: + return resolved + + from litellm.proxy.common_utils.fips import TLS_VERIFICATION_DISABLED, FipsModeError, is_fips_mode + + if is_fips_mode(): + raise FipsModeError(TLS_VERIFICATION_DISABLED) + return resolved + + +def resolve_ssl_verify( + ssl_verify: bool | str | None = None, ) -> bool | str: """ Common utility to resolve the SSL verification setting. diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..2cbe154d2a1 --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,85 @@ +import os +from typing import Final + +from litellm.secret_managers.main import str_to_bool + +FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +TLS_VERIFICATION_DISABLED: Final = ( + f"{FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by SSL_VERIFY, " + "litellm_settings.ssl_verify or a model's ssl_verify. Remove the override, or point ssl_verify at a CA bundle." +) + + +class FipsModeError(Exception): + pass + + +def is_fips_mode() -> bool: + return str_to_bool(os.getenv(FIPS_MODE_ENV_VAR)) is True + + +def openssl_enforces_fips() -> bool: + """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" + try: + import _hashlib + + if _hashlib.get_fips_mode() != 1: + return False + except (ImportError, AttributeError, ValueError): + return False + try: + _hashlib.new("md5", b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def cryptography_enforces_fips() -> bool: + """The cryptography library's own OpenSSL runs with FIPS-only defaults and its FIPS provider can hash SHA-256.""" + try: + from cryptography.exceptions import InternalError, UnsupportedAlgorithm + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + from cryptography.hazmat.primitives import hashes + except ImportError: + return False + try: + if not rust_openssl.is_fips_enabled(): + return False + hashes.Hash(hashes.SHA256()).finalize() + except (AttributeError, InternalError, UnsupportedAlgorithm): + return False + return True + + +def fips_mode_refusal() -> str | None: + """Why the proxy must not start: LITELLM_FIPS_MODE is not a boolean, or is on but TLS verification is off + or Python's or the cryptography library's OpenSSL does not enforce FIPS. None when it may start.""" + value: Final = os.getenv(FIPS_MODE_ENV_VAR) + if value and str_to_bool(value) is None: + return f"{FIPS_MODE_ENV_VAR}={value!r} is not a boolean. Set it to true or false, or unset it." + if not is_fips_mode(): + return None + + from litellm.llms.custom_httpx.http_handler import resolve_ssl_verify + + if resolve_ssl_verify() is False: + return TLS_VERIFICATION_DISABLED + if not openssl_enforces_fips(): + return ( + f"{FIPS_MODE_ENV_VAR} is on but OpenSSL does not enforce FIPS. Run the proxy on an OpenSSL with " + f"its FIPS provider active and default_properties = fips=yes, or unset {FIPS_MODE_ENV_VAR}." + ) + if not cryptography_enforces_fips(): + return ( + f"{FIPS_MODE_ENV_VAR} is on but the cryptography library's OpenSSL has no working FIPS provider. Build it " + "against the system OpenSSL (pip install --no-binary cryptography cryptography), or unset " + f"{FIPS_MODE_ENV_VAR}." + ) + return None + + +def enforce_fips_mode() -> None: + """Raise FipsModeError with the fips_mode_refusal reason, if there is one.""" + refusal: Final = fips_mode_refusal() + if refusal is not None: + raise FipsModeError(refusal) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 64fb2791913..9eda907cf66 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -489,6 +489,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_mode, is_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1531,6 +1532,8 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState if decoded_worker_config is not None: await initialize_from_worker_config(decoded_worker_config) + enforce_fips_mode() + enforce_master_key_boot_verdict( await with_stored_secrets_counted( master_key_boot_verdict( @@ -1573,9 +1576,14 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState result: Final = await migrate_passwords_to_scrypt_async(prisma_client) verbose_proxy_logger.info("Password migration: %s", result) except Exception as e: + if is_fips_mode(): + raise FipsModeError(f"Password migration failed, so plaintext passwords stay unhashed: {e}") from e verbose_proxy_logger.warning("Password migration skipped: %s", e) - asyncio.create_task(_run_pw_migration()) + if is_fips_mode(): + await _run_pw_migration() + else: + asyncio.create_task(_run_pw_migration()) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..0a20d14fb19 --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,64 @@ +import _hashlib +import json +from pathlib import Path +from typing import Final + +import pytest + +from tests.integration._support.client import Gateway +from tests.integration._support.process import owned_proxy, refused_boot_log + + +def _config(directory: Path, upstream_url: str, *, ssl_verify: bool = True) -> Path: + config: Final = directory / "fips_boot.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "fips-model", + "litellm_params": {"model": "openai/fips-model", "api_base": upstream_url, "api_key": "sk-x"}, + } + ], + "litellm_settings": {"ssl_verify": ssl_verify}, + } + ) + ) + return config + + +def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's OpenSSL is in FIPS mode") + log: Final = refused_boot_log( + gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=_config(tmp_path, gateway.upstream_url) + ) + assert "LITELLM_FIPS_MODE is on but OpenSSL does not enforce FIPS" in log, log + + +@pytest.mark.parametrize( + "overrides, ssl_verify", + [({"SSL_VERIFY": "false"}, True), ({}, False)], + ids=["ssl-verify-env", "litellm-settings"], +) +def test_fips_mode_refuses_to_start_with_tls_verification_disabled( + gateway: Gateway, tmp_path: Path, overrides: dict[str, str], ssl_verify: bool +) -> None: + config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=ssl_verify) + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", **overrides}, config=config) + assert "LITELLM_FIPS_MODE is on but TLS certificate verification is disabled" in log, log + + +def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log( + gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}, config=_config(tmp_path, gateway.upstream_url) + ) + assert "LITELLM_FIPS_MODE='enforced' is not a boolean" in log, log + + +def test_fips_mode_off_starts_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: + config: Final = _config(tmp_path, gateway.upstream_url, ssl_verify=False) + with owned_proxy( + gateway, tmp_path, {"SSL_VERIFY": "false"}, config=config, remove_environment=("LITELLM_FIPS_MODE",) + ) as proxy: + assert proxy.client.get("/health/readiness").status_code == 200 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..596d45f88d1 --- /dev/null +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -0,0 +1,144 @@ +import _hashlib +from contextlib import nullcontext +from typing import Final +from unittest.mock import Mock + +import pytest +from cryptography.exceptions import InternalError +from cryptography.hazmat.bindings._rust import openssl as rust_openssl +from cryptography.hazmat.primitives import hashes + +import litellm +from litellm.llms.custom_httpx.http_handler import get_ssl_verify +from litellm.proxy.common_utils.fips import ( + FipsModeError, + cryptography_enforces_fips, + enforce_fips_mode, + is_fips_mode, + openssl_enforces_fips, +) + + +def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: + real_new: Final = _hashlib.new + + def new(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: + if name == "md5" and md5_refused: + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, usedforsecurity=usedforsecurity) + + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: fips_flag) + monkeypatch.setattr(_hashlib, "new", new) + + +@pytest.mark.parametrize("value, expected", [("true", True), (" TRUE ", True), ("false", False), (None, False)]) +def test_is_fips_mode(monkeypatch: pytest.MonkeyPatch, value: str | None, expected: bool) -> None: + if value is None: + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) + else: + monkeypatch.setenv("LITELLM_FIPS_MODE", value) + assert is_fips_mode() is expected + + +@pytest.mark.parametrize( + "fips_flag, md5_refused, enforces", + [(1, True, True), (0, True, False), (1, False, False)], + ids=["fips", "fips-flag-off", "md5-allowed"], +) +def test_openssl_enforces_fips_needs_the_fips_flag_and_a_refused_md5( + monkeypatch: pytest.MonkeyPatch, fips_flag: int, md5_refused: bool, enforces: bool +) -> None: + _fake_openssl(monkeypatch, fips_flag=fips_flag, md5_refused=md5_refused) + assert openssl_enforces_fips() is enforces + + +def test_openssl_without_get_fips_mode_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delattr(_hashlib, "get_fips_mode") + assert openssl_enforces_fips() is False + + +@pytest.mark.parametrize( + "fips_enabled, sha256_broken, enforces", + [(True, False, True), (False, False, False), (True, True, False)], + ids=["fips", "fips-flag-off", "sha256-broken"], +) +def test_cryptography_enforces_fips_needs_the_fips_flag_and_a_working_sha256( + monkeypatch: pytest.MonkeyPatch, fips_enabled: bool, sha256_broken: bool, enforces: bool +) -> None: + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: fips_enabled) + if sha256_broken: + monkeypatch.setattr(hashes, "Hash", Mock(side_effect=InternalError("Unknown OpenSSL error", []))) + assert cryptography_enforces_fips() is enforces + + +def test_cryptography_without_is_fips_enabled_does_not_enforce_fips(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delattr(rust_openssl, "is_fips_enabled") + assert cryptography_enforces_fips() is False + + +def test_this_runners_python_openssl_does_not_enforce_fips() -> None: + if _hashlib.get_fips_mode() == 1: + pytest.skip("this runner's Python OpenSSL is in FIPS mode") + assert openssl_enforces_fips() is False + + +def test_this_runners_cryptography_openssl_does_not_enforce_fips() -> None: + if rust_openssl.is_fips_enabled(): + pytest.skip("this runner's cryptography OpenSSL is in FIPS mode") + assert cryptography_enforces_fips() is False + + +@pytest.mark.parametrize( + "fips_mode, ssl_verify_env, ssl_verify_setting, openssl_fips, cryptography_fips, refusal", + [ + (None, "false", False, False, False, None), + ("", "false", False, False, False, None), + ("false", "false", False, False, False, None), + ("true", None, True, True, True, None), + ("true", "false", True, True, True, "TLS certificate verification is disabled"), + ("true", None, False, True, True, "TLS certificate verification is disabled"), + ("true", None, True, False, True, "is on but OpenSSL does not enforce FIPS"), + ("true", None, True, True, False, "cryptography library's OpenSSL has no working FIPS provider"), + ("enforced", None, True, True, True, "LITELLM_FIPS_MODE='enforced' is not a boolean"), + ], + ids=[ + "unset", + "empty", + "false", + "fips", + "ssl-verify-env-off", + "ssl-verify-setting-off", + "openssl-not-fips", + "cryptography-not-fips", + "not-a-boolean", + ], +) +def test_enforce_fips_mode( + monkeypatch: pytest.MonkeyPatch, + fips_mode: str | None, + ssl_verify_env: str | None, + ssl_verify_setting: bool, + openssl_fips: bool, + cryptography_fips: bool, + refusal: str | None, +) -> None: + for name, value in (("LITELLM_FIPS_MODE", fips_mode), ("SSL_VERIFY", ssl_verify_env)): + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) + monkeypatch.setattr(litellm, "ssl_verify", ssl_verify_setting) + _fake_openssl(monkeypatch, fips_flag=int(openssl_fips), md5_refused=openssl_fips) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: cryptography_fips) + + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + enforce_fips_mode() + + +@pytest.mark.parametrize("fips_mode, refused", [("true", True), ("false", False)], ids=["fips", "fips-off"]) +def test_a_model_that_turns_tls_verification_off_is_refused_in_fips_mode( + monkeypatch: pytest.MonkeyPatch, fips_mode: str, refused: bool +) -> None: + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + with pytest.raises(FipsModeError, match="or a model's ssl_verify") if refused else nullcontext(): + assert get_ssl_verify(ssl_verify=False) is False diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 55d1587669a..4b771a2cebd 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -1778,6 +1778,74 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + config_path, _ = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + config_path.write_text(yaml.dump({"litellm_settings": {"ssl_verify": False}})) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError, match="TLS certificate verification is disabled"): + async with proxy_startup_event(FastAPI()): + pass + + +class _UsersWithOnePlaintextPassword: + async def find_many(self, where: object) -> tuple[types.SimpleNamespace, ...]: + return (types.SimpleNamespace(user_id="user-1", password="a-plaintext-password"),) + + async def update(self, where: object, data: object) -> None: + raise AssertionError("a refused boot must not write to the database") + + +class _DatabaseWithOnePlaintextPassword: + def __init__(self) -> None: + self.db: Final = types.SimpleNamespace(litellm_usertable=_UsersWithOnePlaintextPassword()) + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_password_migration_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + import _hashlib + import hashlib + + from cryptography.hazmat.bindings._rust import openssl as rust_openssl + + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + real_new: Final = _hashlib.new + + def new_refusing_md5(name: str, data: bytes = b"", *, usedforsecurity: bool = True) -> _hashlib.HASH: + if name == "md5": + raise _hashlib.UnsupportedDigestmodError("unsupported hash type") + return real_new(name, data, usedforsecurity=usedforsecurity) + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) + monkeypatch.setattr(_hashlib, "new", new_refusing_md5) + monkeypatch.setattr(rust_openssl, "is_fips_enabled", lambda: True) + scrypt_refused: Final = ValueError("[digital envelope routines] unsupported") + monkeypatch.setattr(hashlib, "scrypt", mock.Mock(side_effect=scrypt_refused)) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _DatabaseWithOnePlaintextPassword()) + + with pytest.raises(FipsModeError, match="plaintext passwords stay unhashed") as refused: + async with proxy_startup_event(FastAPI()): + pass + + assert refused.value.__cause__ is scrypt_refused + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext