From 220dbae71be3a1765c887776daec85814ba3d5eb Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 2 Nov 2026 12:18:10 -0800 Subject: [PATCH] feat(fips): refuse a database that does not match general_settings.fips_mode With fips_mode on, the proxy only starts on a new database or one already set up in FIPS mode, and the first boot marks it. Without fips_mode, a marked database refuses. --- litellm/proxy/common_utils/fips.py | 55 ++++++++ litellm/proxy/proxy_server.py | 9 +- litellm/repositories/fips_repository.py | 81 ++++++++++++ .../configuration/test_fips_mode_boot.py | 25 ++++ .../database/test_fips_database.py | 94 ++++++++++++++ tests/unit/proxy/common_utils/test_fips.py | 117 +++++++++++++++++- ...test_proxy_server_endpoints_and_startup.py | 33 ++++- 7 files changed, 411 insertions(+), 3 deletions(-) create mode 100644 litellm/repositories/fips_repository.py create mode 100644 tests/integration/database/test_fips_database.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 37142364af9..0667143f53c 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -1,6 +1,8 @@ from collections.abc import Mapping from typing import Final +from litellm._logging import verbose_proxy_logger +from litellm.repositories.fips_repository import FIPS_MARKER, FipsRepository from litellm.secret_managers.main import str_to_bool FIPS_MODE_SETTING: Final = "fips_mode" @@ -22,6 +24,10 @@ def _fips_mode_setting(general_settings: Mapping[str, object]) -> bool | None: return str_to_bool(value) if isinstance(value, str) else None +def is_fips_mode(general_settings: Mapping[str, object]) -> bool: + return _fips_mode_setting(general_settings) is True + + def openssl_enforces_fips() -> bool: """Python's OpenSSL runs with FIPS-only defaults and itself refuses MD5.""" try: @@ -88,3 +94,52 @@ def enforce_fips_mode(general_settings: Mapping[str, object]) -> None: refusal: Final = fips_mode_refusal(general_settings) if refusal is not None: raise FipsModeError(refusal) + + +async def _is_marked_fips(database: FipsRepository) -> bool: + try: + return await database.is_marked_fips() + except Exception as error: # noqa: BLE001 # without FIPS mode, an unreadable database starts as it did before + verbose_proxy_logger.warning("Could not check whether the database was set up in FIPS mode: %s", error) + return False + + +async def fips_database_refusal( + database: FipsRepository | None, *, fips_mode: bool, database_is_configured: bool +) -> str | None: + """Why the proxy must not start on its database: FIPS mode is off on a database set up in FIPS mode, or on over + a database that can't be checked or already holds rows written without it. None when it may start.""" + if not fips_mode: + if database is not None and await _is_marked_fips(database): + return ( + f"This database was set up with {_SETTING}: true, so the proxy only starts on it in FIPS mode. " + f"To stop using FIPS mode on it for good, run " + f"DELETE FROM \"LiteLLM_Config\" WHERE param_name = '{FIPS_MARKER}'." + ) + return None + if database is None: + return f"{_SETTING} is on but the database can't be reached to check it." if database_is_configured else None + try: + state: Final = await database.state() + except Exception as error: # noqa: BLE001 # a value, so FIPS mode refuses a database it can't check + return f"{_SETTING} is on but the database could not be checked: {error}" + if state.holds_rows and not state.marked_fips: + return ( + f"{_SETTING} is on but this database already holds data written without it. " + "FIPS mode only starts on a new, empty database." + ) + return None + + +async def enforce_fips_database( + database: FipsRepository | None, *, fips_mode: bool, database_is_configured: bool +) -> None: + """Raise FipsModeError with the fips_database_refusal reason, if there is one, then mark the database as set up + in FIPS mode when FIPS mode is on.""" + refusal: Final = await fips_database_refusal( + database, fips_mode=fips_mode, database_is_configured=database_is_configured + ) + if refusal is not None: + raise FipsModeError(refusal) + if database is not None and fips_mode: + await database.mark_fips() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index baaeced2375..7bf5e07e4e6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -499,7 +499,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id -from litellm.proxy.common_utils.fips import enforce_fips_mode +from litellm.proxy.common_utils.fips import enforce_fips_database, enforce_fips_mode, is_fips_mode from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -992,6 +992,7 @@ from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports from litellm.proxy.video_endpoints.endpoints import router as video_router from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.credentials_repository import CredentialsRepository +from litellm.repositories.fips_repository import FipsRepository from litellm.repositories.prisma_protocols import TableActions from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.router import ( @@ -1640,6 +1641,12 @@ async def _proxy_lifespan(app: FastAPI, resources: AsyncExitStack) -> AsyncGener database: Final = existing_database or await _own_database(resources, prisma_client) + await enforce_fips_database( + None if prisma_client is None else FipsRepository(prisma_client), + fips_mode=is_fips_mode(general_settings_view()), + database_is_configured=get_secret("DATABASE_URL", None) is not None, + ) + await migrate_if_requested( environ=os.environ, master_key=master_key, diff --git a/litellm/repositories/fips_repository.py b/litellm/repositories/fips_repository.py new file mode 100644 index 00000000000..15c80272483 --- /dev/null +++ b/litellm/repositories/fips_repository.py @@ -0,0 +1,81 @@ +"""FIPS repository for the LiteLLM_Config row that marks a database as set up in FIPS mode.""" + +from dataclasses import dataclass +from typing import Final, Protocol, cast # noqa: TID251 # Prisma delegates query_raw and execute_raw dynamically + +from litellm.proxy.db.create_views import SupportsRawQueries +from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry + +FIPS_MARKER: Final = "fips_mode" +_IS_MARKED: Final = 'SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1' +_OTHER_TABLES: Final = ( + "SELECT table_name FROM information_schema.tables WHERE table_schema = ANY (current_schemas(false)) " + "AND table_type = 'BASE TABLE' AND table_name LIKE 'LiteLLM\\_%' AND table_name <> 'LiteLLM_Config'" +) +_MARK: Final = ( + "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ($1, 'true'::jsonb) " + "ON CONFLICT (param_name) DO NOTHING" +) + + +def _quoted(identifier: str) -> str: + return '"' + identifier.replace('"', '""') + '"' + + +class _WithWriter(Protocol): + @property + def writer_db(self) -> object: ... + + async def attempt_db_reconnect( + self, *, reason: str, timeout_seconds: float | None = None, lock_timeout_seconds: float | None = None + ) -> bool: ... + + +@dataclass(frozen=True, slots=True) +class FipsDatabaseState: + marked_fips: bool + holds_rows: bool + + +class FipsRepository: + """Reads and writes go to the writer, since a replica may not have the marker yet. A read that loses its + connection reconnects and runs once more.""" + + def __init__(self, prisma_client: _WithWriter) -> None: + self._prisma_client: Final = prisma_client + + @property + def _writer(self) -> SupportsRawQueries: + writer: Final = self._prisma_client.writer_db + return cast(SupportsRawQueries, writer) # cast-ok: Prisma delegates database methods dynamically + + async def is_marked_fips(self) -> bool: + return await call_with_db_reconnect_retry( + self._prisma_client, self._read_marker, reason="fips_startup_read_marker_failure" + ) + + async def state(self) -> FipsDatabaseState: + """The marker and whether any other LiteLLM row exists, read in one statement so they agree.""" + return await call_with_db_reconnect_retry( + self._prisma_client, self._read_state, reason="fips_startup_read_state_failure" + ) + + async def _read_marker(self) -> bool: + return bool(await self._writer.query_raw(_IS_MARKED, FIPS_MARKER)) + + async def _read_state(self) -> FipsDatabaseState: + writer: Final = self._writer + tables: Final = tuple(str(row["table_name"]) for row in await writer.query_raw(_OTHER_TABLES)) + any_row_in_other_tables: Final = "".join(f" OR EXISTS (SELECT 1 FROM {_quoted(table)})" for table in tables) + row: Final = ( + await writer.query_raw( + 'SELECT EXISTS (SELECT 1 FROM "LiteLLM_Config" WHERE param_name = $1) AS marked_fips, ' + f'(EXISTS (SELECT 1 FROM "LiteLLM_Config" WHERE param_name <> $1){any_row_in_other_tables}) ' + "AS holds_rows", + FIPS_MARKER, + ) + )[0] + return FipsDatabaseState(marked_fips=row["marked_fips"] is True, holds_rows=row["holds_rows"] is True) + + async def mark_fips(self) -> None: + await self._writer.execute_raw(_MARK, FIPS_MARKER) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py index 18fb692da6e..b4637043800 100644 --- a/tests/integration/configuration/test_fips_mode_boot.py +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -6,6 +6,7 @@ from typing import Final import pytest from tests.integration._support.client import Gateway +from tests.integration._support.database import scratch_database, write_rows from tests.integration._support.process import refused_boot_log @@ -37,3 +38,27 @@ def test_fips_mode_refuses_to_start_when_openssl_does_not_enforce_fips(gateway: def test_fips_mode_refuses_to_start_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: log: Final = refused_boot_log(gateway, tmp_path, {}, config=_config(tmp_path, gateway.upstream_url, "enforced")) assert "general_settings.fips_mode='enforced' is not a boolean" in log, log + + +def test_proxy_refuses_to_start_without_fips_mode_on_a_database_set_up_in_fips_mode( + gateway: Gateway, tmp_path: Path +) -> None: + with scratch_database() as database_url: + write_rows( + 'CREATE TABLE "LiteLLM_Config" (param_name TEXT NOT NULL PRIMARY KEY, param_value JSONB, ' + "last_run_at TIMESTAMP(3), reload_revision BIGINT NOT NULL DEFAULT 0)", + (), + database_url=database_url, + ) + write_rows( + "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('fips_mode', 'true')", + (), + database_url=database_url, + ) + log: Final = refused_boot_log( + gateway, + tmp_path, + {"DATABASE_URL": database_url}, + config=_config(tmp_path, gateway.upstream_url, False), + ) + assert "This database was set up with general_settings.fips_mode: true" in log, log diff --git a/tests/integration/database/test_fips_database.py b/tests/integration/database/test_fips_database.py new file mode 100644 index 00000000000..36074e48719 --- /dev/null +++ b/tests/integration/database/test_fips_database.py @@ -0,0 +1,94 @@ +import asyncio +from contextlib import nullcontext +from typing import Final, LiteralString + +import pytest + +from litellm.proxy.common_utils.fips import FipsModeError, enforce_fips_database +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.utils import PrismaClient, ProxyLogging +from litellm.repositories.fips_repository import FipsRepository +from tests.integration._support.database import read_rows, scratch_database, write_rows + +_CONFIG_TABLE: Final = ( + 'CREATE TABLE "LiteLLM_Config" (param_name TEXT PRIMARY KEY, param_value JSONB, last_run_at TIMESTAMP(3), ' + "reload_revision BIGINT NOT NULL DEFAULT 0)" +) +_KEY_TABLE: Final = 'CREATE TABLE "LiteLLM_VerificationToken" (token TEXT PRIMARY KEY)' +_MARKER: Final = "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('fips_mode', 'true')" +_KEY: Final = "INSERT INTO \"LiteLLM_VerificationToken\" (token) VALUES ('hashed-key')" +_SETTING: Final = "INSERT INTO \"LiteLLM_Config\" (param_name, param_value) VALUES ('general_settings', '{}')" + + +def _marker_rows(database_url: str) -> int: + return len( + read_rows('SELECT 1 FROM "LiteLLM_Config" WHERE param_name = %s', ("fips_mode",), database_url=database_url) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, setup, refusal, marked_after", + [ + (True, (_CONFIG_TABLE, _KEY_TABLE), None, 1), + (True, (_CONFIG_TABLE, _KEY_TABLE, _MARKER, _KEY), None, 1), + (True, (_CONFIG_TABLE, _KEY_TABLE, _KEY), "already holds data written without it", 0), + (True, (_CONFIG_TABLE, _KEY_TABLE, _SETTING), "already holds data written without it", 0), + (True, (), "could not be checked", None), + (False, (_CONFIG_TABLE, _KEY_TABLE, _MARKER), "set up with general_settings.fips_mode: true", 1), + (False, (_CONFIG_TABLE, _KEY_TABLE, _KEY), None, 0), + (False, (), None, None), + ], + ids=[ + "fips-on-new-database", + "fips-on-fips-database", + "fips-on-database-with-a-key", + "fips-on-database-with-a-setting", + "fips-on-database-without-tables", + "fips-off-fips-database", + "fips-off-database-with-a-key", + "fips-off-database-without-tables", + ], +) +async def test_fips_database_check( + monkeypatch: pytest.MonkeyPatch, + fips_mode: bool, + setup: tuple[LiteralString, ...], + refusal: str | None, + marked_after: int | None, +) -> None: + with scratch_database() as database_url: + for statement in setup: + write_rows(statement, (), database_url=database_url) + monkeypatch.setenv("DATABASE_URL", database_url) + client: Final = PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) + await client.connect() + try: + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(FipsRepository(client), fips_mode=fips_mode, database_is_configured=True) + finally: + await client.aclose() + if marked_after is not None: + assert _marker_rows(database_url) == marked_after + + +@pytest.mark.asyncio +async def test_pods_starting_together_in_fips_mode_on_a_new_database_both_start( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with scratch_database() as database_url: + for statement in (_CONFIG_TABLE, _KEY_TABLE): + write_rows(statement, (), database_url=database_url) + monkeypatch.setenv("DATABASE_URL", database_url) + pods: Final = tuple(PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) for _ in range(4)) + await asyncio.gather(*(pod.connect() for pod in pods)) + try: + await asyncio.gather( + *( + enforce_fips_database(FipsRepository(pod), fips_mode=True, database_is_configured=True) + for pod in pods + ) + ) + finally: + await asyncio.gather(*(pod.aclose() for pod in pods)) + assert _marker_rows(database_url) == 1 diff --git a/tests/unit/proxy/common_utils/test_fips.py b/tests/unit/proxy/common_utils/test_fips.py index a72772df492..708aedb42e7 100644 --- a/tests/unit/proxy/common_utils/test_fips.py +++ b/tests/unit/proxy/common_utils/test_fips.py @@ -1,9 +1,10 @@ import _hashlib -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from contextlib import nullcontext from typing import Final from unittest.mock import Mock +import httpx import pytest from cryptography.exceptions import InternalError from cryptography.hazmat.bindings._rust import openssl as rust_openssl @@ -12,9 +13,12 @@ from cryptography.hazmat.primitives import hashes from litellm.proxy.common_utils.fips import ( FipsModeError, cryptography_enforces_fips, + enforce_fips_database, enforce_fips_mode, + is_fips_mode, openssl_enforces_fips, ) +from litellm.repositories.fips_repository import FipsRepository def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refused: bool) -> None: @@ -29,6 +33,21 @@ def _fake_openssl(monkeypatch: pytest.MonkeyPatch, *, fips_flag: int, md5_refuse monkeypatch.setattr(_hashlib, "new", new) +@pytest.mark.parametrize( + "general_settings, expected", + [ + ({"fips_mode": True}, True), + ({"fips_mode": " TRUE "}, True), + ({"fips_mode": False}, False), + ({"fips_mode": "false"}, False), + ({"fips_mode": None}, False), + ({}, False), + ], +) +def test_is_fips_mode(general_settings: Mapping[str, object], expected: bool) -> None: + assert is_fips_mode(general_settings) is expected + + @pytest.mark.parametrize( "fips_flag, md5_refused, enforces", [(1, True, True), (0, True, False), (1, False, False)], @@ -118,3 +137,99 @@ def test_enforce_fips_mode( with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): enforce_fips_mode(general_settings) + + +class _Database: + """Prisma's raw-query edge over a database whose only LiteLLM table besides LiteLLM_Config is the key table.""" + + def __init__(self, *, marked_fips: bool, holds_rows: bool, readable: bool = True, dropped_reads: int = 0) -> None: + self.marked_fips = marked_fips + self.holds_rows = holds_rows + self.readable = readable + self.dropped_reads = dropped_reads + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + if self.dropped_reads: + self.dropped_reads -= 1 + raise httpx.ReadError("connection dropped") + if not self.readable: + raise RuntimeError('relation "LiteLLM_Config" does not exist') + if "information_schema.tables" in query: + return [{"table_name": "LiteLLM_VerificationToken"}] + if '"LiteLLM_VerificationToken"' in query: + return [{"marked_fips": self.marked_fips, "holds_rows": self.holds_rows}] + return [{"?column?": 1}] if self.marked_fips else [] + + async def execute_raw(self, query: str, *args: object) -> int: + self.marked_fips = True + return 1 + + +class _Prisma: + def __init__(self, database: _Database) -> None: + self.writer_db: Final = database + + async def attempt_db_reconnect( + self, *, reason: str, timeout_seconds: float | None = None, lock_timeout_seconds: float | None = None + ) -> bool: + return True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, database, refusal, marked_after", + [ + (True, _Database(marked_fips=False, holds_rows=False), None, True), + (True, _Database(marked_fips=True, holds_rows=True), None, True), + (True, _Database(marked_fips=False, holds_rows=True), "already holds data written without it", False), + (True, _Database(marked_fips=False, holds_rows=False, readable=False), "could not be checked", False), + (False, _Database(marked_fips=True, holds_rows=True), "set up with general_settings.fips_mode: true", True), + (False, _Database(marked_fips=False, holds_rows=True), None, False), + (False, _Database(marked_fips=True, holds_rows=True, readable=False), None, True), + ( + True, + _Database(marked_fips=False, holds_rows=True, dropped_reads=1), + "already holds data written without it", + False, + ), + ( + False, + _Database(marked_fips=True, holds_rows=True, dropped_reads=1), + "set up with general_settings.fips_mode: true", + True, + ), + (False, _Database(marked_fips=True, holds_rows=True, dropped_reads=2), None, True), + ], + ids=[ + "fips-on-new-database", + "fips-on-fips-database", + "fips-on-used-database", + "fips-on-unreadable-database", + "fips-off-fips-database", + "fips-off-used-database", + "fips-off-unreadable-database", + "fips-on-used-database-after-a-dropped-connection", + "fips-off-fips-database-after-a-dropped-connection", + "fips-off-fips-database-dropped-twice", + ], +) +async def test_enforce_fips_database( + fips_mode: bool, database: _Database, refusal: str | None, marked_after: bool +) -> None: + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(FipsRepository(_Prisma(database)), fips_mode=fips_mode, database_is_configured=True) + + assert database.marked_fips is marked_after + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fips_mode, database_is_configured, refusal", + [(True, True, "can't be reached"), (True, False, None), (False, True, None)], + ids=["fips-on-unreachable", "fips-on-no-database", "fips-off-unreachable"], +) +async def test_enforce_fips_database_without_a_connected_database( + fips_mode: bool, database_is_configured: bool, refusal: str | None +) -> None: + with pytest.raises(FipsModeError, match=refusal) if refusal else nullcontext(): + await enforce_fips_database(None, fips_mode=fips_mode, database_is_configured=database_is_configured) diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 8def4b0541a..c46c1b336ae 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -10,7 +10,7 @@ import subprocess import sys import time import types -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -1790,6 +1790,37 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +class _DatabaseSetUpInFipsMode: + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + return [{"?column?": 1}] + + async def execute_raw(self, query: str, *args: object) -> int: + raise AssertionError("a refused boot must not write to the database") + + +class _PrismaClientOnAFipsDatabase: + def __init__(self) -> None: + self.writer_db: Final = _DatabaseSetUpInFipsMode() + + async def aclose(self) -> None: + pass + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_a_database_set_up_in_fips_mode_when_fips_mode_is_off( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _PrismaClientOnAFipsDatabase()) + + with pytest.raises(FipsModeError, match=r"set up with general_settings\.fips_mode: true"): + async with proxy_startup_event(FastAPI()): + pass + + @pytest.mark.asyncio async def test_proxy_startup_reads_fips_mode_from_the_config_file( monkeypatch: pytest.MonkeyPatch, tmp_path: Path