From 67895955f6e9f40e78a96c89a642adc03b5d6a3c Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:20:08 +0000 Subject: [PATCH 01/17] feat(proxy): LITELLM_FIPS_MODE startup gate with provider assertion, TLS verify guard and loud password migration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 151 ++++++++++++++++++ litellm/proxy/proxy_server.py | 31 +++- tests/integration/_support/process.py | 76 +++++++-- .../configuration/test_fips_mode_boot.py | 66 ++++++++ tests/integration/contracts.json | 15 ++ .../proxy/common_utils/test_fips.py | 118 ++++++++++++++ tests/test_litellm/proxy/test_proxy_server.py | 104 ++++++++++++ 7 files changed, 546 insertions(+), 15 deletions(-) create mode 100644 litellm/proxy/common_utils/fips.py create mode 100644 tests/integration/configuration/test_fips_mode_boot.py create mode 100644 tests/test_litellm/proxy/common_utils/test_fips.py diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py new file mode 100644 index 00000000000..33f35388567 --- /dev/null +++ b/litellm/proxy/common_utils/fips.py @@ -0,0 +1,151 @@ +import hashlib +import os +from collections.abc import Callable +from dataclasses import dataclass +from typing import Final + +from typing_extensions import assert_never + +FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" +SSL_VERIFY_ENV_VAR: Final = "SSL_VERIFY" +SSL_VERIFY_SETTING: Final = "litellm_settings.ssl_verify" +REFUSAL_PREFIX: Final = "LiteLLM proxy refused to start" + +_TRUE_VALUES: Final = frozenset({"true", "1", "yes", "on"}) +_FALSE_VALUES: Final = frozenset({"false", "0", "no", "off", ""}) + + +@dataclass(frozen=True, slots=True) +class FipsModeOff: + pass + + +@dataclass(frozen=True, slots=True) +class FipsModeOn: + pass + + +@dataclass(frozen=True, slots=True) +class MalformedFipsMode: + value: str + + +FipsModeSetting = FipsModeOff | FipsModeOn | MalformedFipsMode + + +@dataclass(frozen=True, slots=True) +class ProviderDoesNotEnforceFips: + pass + + +@dataclass(frozen=True, slots=True) +class TlsVerificationDisabled: + sources: tuple[str, ...] + + +FipsBootRefusal = MalformedFipsMode | ProviderDoesNotEnforceFips | TlsVerificationDisabled +FipsBootVerdict = FipsModeOff | FipsModeOn | FipsBootRefusal + + +class FipsModeError(Exception): + pass + + +def parse_fips_mode(raw: str | None) -> FipsModeSetting: + if raw is None: + return FipsModeOff() + normalized: Final = raw.strip().lower() + if normalized in _TRUE_VALUES: + return FipsModeOn() + if normalized in _FALSE_VALUES: + return FipsModeOff() + return MalformedFipsMode(value=raw) + + +def is_fips_mode(environ: Callable[[str], str | None] = os.environ.get) -> bool: + return isinstance(parse_fips_mode(environ(FIPS_MODE_ENV_VAR)), FipsModeOn) + + +def openssl_enforces_fips() -> bool: + """MD5 is not an approved digest, so an enforcing FIPS provider refuses it even when asked for security use.""" + try: + hashlib.md5(b"", usedforsecurity=True) + except ValueError: + return True + return False + + +def fips_boot_verdict( + *, + raw_fips_mode: str | None, + provider_enforces_fips: Callable[[], bool], + ssl_verify_environment: str | None, + ssl_verify_setting: object, +) -> FipsBootVerdict: + setting: Final = parse_fips_mode(raw_fips_mode) + match setting: + case FipsModeOff() | MalformedFipsMode(): + return setting + case FipsModeOn(): + pass + case _: + assert_never(setting) + disabled: Final = tuple( + source + for source, off in ( + (SSL_VERIFY_ENV_VAR, _is_off(ssl_verify_environment)), + (SSL_VERIFY_SETTING, _is_off(ssl_verify_setting)), + ) + if off + ) + if disabled: + return TlsVerificationDisabled(sources=disabled) + if not provider_enforces_fips(): + return ProviderDoesNotEnforceFips() + return setting + + +def enforce_fips_boot_verdict(verdict: FipsBootVerdict, announce: Callable[[str], object]) -> None: + match verdict: + case FipsModeOff() | FipsModeOn(): + return + case MalformedFipsMode() | ProviderDoesNotEnforceFips() | TlsVerificationDisabled(): + message: Final = render_refusal(verdict) + announce(f"\n{message}\n\n") + raise FipsModeError(message) + case _: + assert_never(verdict) + + +def render_refusal(refusal: FipsBootRefusal) -> str: + match refusal: + case MalformedFipsMode(value=value): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={value} is not a boolean.\n" + f"Set {FIPS_MODE_ENV_VAR} to true or false, or unset it." + ) + case ProviderDoesNotEnforceFips(): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but this Python does not enforce FIPS.\n" + "Its OpenSSL still allows non-approved algorithms (MD5 succeeded), so passwords and keys would be\n" + "protected with algorithms the FIPS 140-3 policy forbids. Run the proxy from a FIPS image whose\n" + f"OpenSSL FIPS provider is enabled, or unset {FIPS_MODE_ENV_VAR} on a non-FIPS runtime." + ) + case TlsVerificationDisabled(sources=sources): + return ( + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by " + f"{' and '.join(sources)}.\nFIPS deployments must verify upstream certificates, so remove the " + "override or point ssl_verify at a CA bundle instead." + ) + case _: + assert_never(refusal) + + +def _is_off(value: object) -> bool: + match value: + case bool(): + return value is False + case str(): + return value.strip().lower() in _FALSE_VALUES - {""} + case _: + return False diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index dab7decd4dc..7e2e063ae88 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -404,6 +404,14 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id +from litellm.proxy.common_utils.fips import ( + FIPS_MODE_ENV_VAR, + SSL_VERIFY_ENV_VAR, + enforce_fips_boot_verdict, + fips_boot_verdict, + is_fips_mode, + openssl_enforces_fips, +) from litellm.proxy.common_utils.healthy_model_filter import ( get_hidden_unhealthy_model_names, is_healthy_only_listing_default, @@ -1277,6 +1285,16 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: if isinstance(worker_config, dict): await initialize_from_worker_config(worker_config) + enforce_fips_boot_verdict( + fips_boot_verdict( + raw_fips_mode=os.getenv(FIPS_MODE_ENV_VAR), + provider_enforces_fips=openssl_enforces_fips, + ssl_verify_environment=os.getenv(SSL_VERIFY_ENV_VAR), + ssl_verify_setting=litellm.ssl_verify, + ), + announce=announce_on_stderr_at_exit, + ) + enforce_master_key_boot_verdict( await with_stored_secrets_counted( master_key_boot_verdict( @@ -1316,10 +1334,21 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]: try: result: Final = await migrate_passwords_to_scrypt_async(prisma_client) verbose_proxy_logger.info("Password migration: %s", result) + except ValueError as e: + verbose_proxy_logger.error( + "Password migration failed, so plaintext passwords stay unhashed in the database: %s. " + "This is what an OpenSSL FIPS provider reports when the hashing algorithm is not approved.", + e, + ) + if is_fips_mode(): + raise except Exception as e: verbose_proxy_logger.warning("Password migration skipped: %s", e) - asyncio.create_task(_run_pw_migration()) + if is_fips_mode(): + await _run_pw_migration() + else: + asyncio.create_task(_run_pw_migration()) async def _run_agent_grant_id_migration() -> None: from litellm.proxy.agent_endpoints.agent_registry import ( diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 5c44beaa570..6dec960097b 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -68,8 +68,15 @@ def owned_proxy( yield owned.gateway +@dataclass(frozen=True, slots=True) +class LaunchedProxy: + port: int + process: subprocess.Popen[bytes] + log: Path + + @contextmanager -def owned_proxy_process( +def launched_proxy( gateway: Gateway, directory: Path, overrides: Mapping[str, str], @@ -77,7 +84,7 @@ def owned_proxy_process( config: Path | None = None, remove_environment: tuple[str, ...] = (), workers: int = 1, -) -> Iterator[OwnedProxy]: +) -> Iterator[LaunchedProxy]: with socket.socket() as reserve: reserve.bind(("127.0.0.1", 0)) port: Final = reserve.getsockname()[1] @@ -116,18 +123,7 @@ def owned_proxy_process( start_new_session=True, ) try: - with httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=15, trust_env=False) as client: - deadline: Final = time.monotonic() + 70 - while True: - assert process.poll() is None, "Owned proxy exited before readiness" - try: - if client.get("/health/readiness", timeout=2).status_code == 200: - break - except httpx.TransportError: - pass - assert time.monotonic() < deadline, "Owned proxy readiness deadline exceeded" - time.sleep(0.1) - yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), process, log_path) + yield LaunchedProxy(port, process, log_path) finally: root_stopped: Final = stop_root_process(process) residual: Final = group_members(process.pid) @@ -142,3 +138,55 @@ def owned_proxy_process( survivors: Final = group_members(process.pid) assert not survivors, "Owned proxy child survived cleanup" assert root_stopped and not remaining, "Owned proxy required forced cleanup" + + +def refused_boot_log( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, +) -> str: + """Start the proxy and return its log once it exits non-zero instead of becoming ready.""" + with launched_proxy(gateway, directory, overrides, config=config) as launched: + with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: + deadline: Final = time.monotonic() + 70 + while launched.process.poll() is None: + try: + ready: Final = client.get("/health/readiness", timeout=2).status_code == 200 + except httpx.TransportError: + ready = False + assert not ready, f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + assert time.monotonic() < deadline, "Proxy neither exited nor became ready within the deadline" + time.sleep(0.1) + assert launched.process.returncode != 0, ( + f"Proxy exited 0 instead of refusing to boot:\n{launched.log.read_text()}" + ) + return launched.log.read_text() + + +@contextmanager +def owned_proxy_process( + gateway: Gateway, + directory: Path, + overrides: Mapping[str, str], + *, + config: Path | None = None, + remove_environment: tuple[str, ...] = (), + workers: int = 1, +) -> Iterator[OwnedProxy]: + with launched_proxy( + gateway, directory, overrides, config=config, remove_environment=remove_environment, workers=workers + ) as launched: + with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: + deadline: Final = time.monotonic() + 70 + while True: + assert launched.process.poll() is None, "Owned proxy exited before readiness" + try: + if client.get("/health/readiness", timeout=2).status_code == 200: + break + except httpx.TransportError: + pass + assert time.monotonic() < deadline, "Owned proxy readiness deadline exceeded" + time.sleep(0.1) + yield OwnedProxy(Gateway(client, gateway.key, gateway.upstream_url), launched.process, launched.log) diff --git a/tests/integration/configuration/test_fips_mode_boot.py b/tests/integration/configuration/test_fips_mode_boot.py new file mode 100644 index 00000000000..5cf69f21ab3 --- /dev/null +++ b/tests/integration/configuration/test_fips_mode_boot.py @@ -0,0 +1,66 @@ +"""LITELLM_FIPS_MODE is a boot gate: the proxy refuses to serve unless the process really enforces FIPS. + +Every leg launches the real proxy binary against the suite's Postgres and asserts on what an operator sees: +exit status and the refusal text in the log. Nothing is patched inside the proxy. +""" + +import hashlib +from pathlib import Path +from typing import Final + +import pytest +import yaml + +from tests.integration._support.client import Gateway +from tests.integration._support.process import owned_proxy, refused_boot_log + +REFUSAL: Final = "LiteLLM proxy refused to start" + + +def _this_python_enforces_fips() -> bool: + try: + hashlib.md5(b"probe", usedforsecurity=True) + except ValueError: + return True + return False + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_when_provider_does_not_enforce_fips") +def test_fips_mode_refuses_to_serve_when_this_python_does_not_enforce_fips(gateway: Gateway, tmp_path: Path) -> None: + if _this_python_enforces_fips(): + pytest.skip("Runner OpenSSL enforces FIPS, so this leg cannot observe the non-enforcing refusal") + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}) + assert REFUSAL in log, log + assert "LITELLM_FIPS_MODE" in log and "does not enforce FIPS" in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled") +@pytest.mark.parametrize("source", ("environment", "config")) +def test_fips_mode_refuses_to_serve_with_tls_verification_disabled( + gateway: Gateway, tmp_path: Path, source: str +) -> None: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + path: Final = tmp_path / "ssl_verify_off.yaml" + settings: Final = {**config.get("litellm_settings", {}), "ssl_verify": False} + path.write_text(yaml.safe_dump({**config, "litellm_settings": settings})) + log: Final = ( + refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true", "SSL_VERIFY": "false"}) + if source == "environment" + else refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "true"}, config=path) + ) + assert REFUSAL in log, log + assert "TLS certificate verification is disabled" in log, log + assert ("SSL_VERIFY" if source == "environment" else "litellm_settings.ssl_verify") in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.refuses_boot_on_malformed_value") +def test_fips_mode_refuses_to_serve_on_a_value_that_is_not_a_boolean(gateway: Gateway, tmp_path: Path) -> None: + log: Final = refused_boot_log(gateway, tmp_path, {"LITELLM_FIPS_MODE": "enforced"}) + assert REFUSAL in log, log + assert "LITELLM_FIPS_MODE=enforced" in log and "true or false" in log, log + + +@pytest.mark.covers("other.configuration.fips_mode.off_leaves_boot_unchanged") +def test_fips_mode_off_serves_even_with_tls_verification_disabled(gateway: Gateway, tmp_path: Path) -> None: + with owned_proxy(gateway, tmp_path, {"LITELLM_FIPS_MODE": "false", "SSL_VERIFY": "false"}) as candidate: + assert candidate.client.get("/health/readiness").status_code == 200 diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index a8d9cf1df8b..673f62fd731 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -76,6 +76,21 @@ "tests/integration/configuration/test_effective_settings.py::test_credential_value_update_and_model_reload_reach_provider": [ "mgmt.credential.update.saved_value_reaches_wire" ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_when_this_python_does_not_enforce_fips": [ + "other.configuration.fips_mode.refuses_boot_when_provider_does_not_enforce_fips" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_with_tls_verification_disabled[environment]": [ + "other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_with_tls_verification_disabled[config]": [ + "other.configuration.fips_mode.refuses_boot_with_tls_verification_disabled" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_refuses_to_serve_on_a_value_that_is_not_a_boolean": [ + "other.configuration.fips_mode.refuses_boot_on_malformed_value" + ], + "tests/integration/configuration/test_fips_mode_boot.py::test_fips_mode_off_serves_even_with_tls_verification_disabled": [ + "other.configuration.fips_mode.off_leaves_boot_unchanged" + ], "tests/integration/management/test_partial_update_sequences.py::test_denied_key_update_preserves_saved_grants_and_serving": [ "mgmt.key.update.denied_request_preserves_effective_state" ], diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py new file mode 100644 index 00000000000..e934cafbdda --- /dev/null +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -0,0 +1,118 @@ +import hashlib + +import pytest + +from litellm.proxy.common_utils.fips import ( + FipsModeError, + FipsModeOff, + FipsModeOn, + MalformedFipsMode, + ProviderDoesNotEnforceFips, + TlsVerificationDisabled, + enforce_fips_boot_verdict, + fips_boot_verdict, + is_fips_mode, + openssl_enforces_fips, + parse_fips_mode, +) + + +def _verdict( + raw: str | None, + *, + enforcing: bool = True, + ssl_env: str | None = None, + ssl_setting: object = True, +): + return fips_boot_verdict( + raw_fips_mode=raw, + provider_enforces_fips=lambda: enforcing, + ssl_verify_environment=ssl_env, + ssl_verify_setting=ssl_setting, + ) + + +@pytest.mark.parametrize("raw", [None, "false", "0", "no", "off", "", " False "]) +def test_unset_and_false_spellings_leave_fips_mode_off(raw): + assert parse_fips_mode(raw) == FipsModeOff() + assert is_fips_mode({"LITELLM_FIPS_MODE": raw}.get) is False + + +@pytest.mark.parametrize("raw", ["true", "1", "yes", "on", " TRUE "]) +def test_true_spellings_turn_fips_mode_on(raw): + assert parse_fips_mode(raw) == FipsModeOn() + assert is_fips_mode({"LITELLM_FIPS_MODE": raw}.get) is True + + +@pytest.mark.parametrize("raw", ["enforced", "2", "strict", "yes please"]) +def test_anything_else_is_malformed_and_refused_with_the_offending_value(raw): + assert parse_fips_mode(raw) == MalformedFipsMode(value=raw) + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(_verdict(raw, enforcing=False), announce=lambda _: None) + assert f"LITELLM_FIPS_MODE={raw} is not a boolean" in str(refused.value) + assert "true or false" in str(refused.value) + + +def test_off_never_consults_the_provider_or_tls_settings(): + def explode() -> bool: + raise AssertionError("provider probe must not run while FIPS mode is off") + + verdict = fips_boot_verdict( + raw_fips_mode=None, provider_enforces_fips=explode, ssl_verify_environment="false", ssl_verify_setting=False + ) + assert verdict == FipsModeOff() + enforce_fips_boot_verdict(verdict, announce=lambda _: pytest.fail("nothing to announce when off")) + + +def test_on_with_an_enforcing_provider_and_verified_tls_boots(): + verdict = _verdict("true", enforcing=True, ssl_env="true", ssl_setting="/etc/ssl/certs/ca.pem") + assert verdict == FipsModeOn() + enforce_fips_boot_verdict(verdict, announce=lambda _: pytest.fail("nothing to announce when on")) + + +def test_on_with_a_non_enforcing_provider_is_refused_and_names_the_fix(): + announced = [] + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(_verdict("true", enforcing=False), announce=announced.append) + message = str(refused.value) + assert message.startswith("LiteLLM proxy refused to start") + assert "LITELLM_FIPS_MODE is on but this Python does not enforce FIPS" in message + assert "FIPS image" in message + assert announced == [f"\n{message}\n\n"] + + +@pytest.mark.parametrize( + "ssl_env, ssl_setting, sources", + [ + ("false", True, ("SSL_VERIFY",)), + ("0", True, ("SSL_VERIFY",)), + (None, False, ("litellm_settings.ssl_verify",)), + (None, "False", ("litellm_settings.ssl_verify",)), + ("no", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), + ], +) +def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_setting, sources): + verdict = _verdict("true", enforcing=True, ssl_env=ssl_env, ssl_setting=ssl_setting) + assert verdict == TlsVerificationDisabled(sources=sources) + with pytest.raises(FipsModeError) as refused: + enforce_fips_boot_verdict(verdict, announce=lambda _: None) + assert "TLS certificate verification is disabled by " + " and ".join(sources) in str(refused.value) + + +@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, ""]) +def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_setting): + assert _verdict("true", enforcing=True, ssl_setting=ssl_setting) == FipsModeOn() + + +def test_disabled_tls_is_reported_before_the_provider_so_operators_see_config_mistakes_first(): + assert _verdict("true", enforcing=False, ssl_env="false") == TlsVerificationDisabled(sources=("SSL_VERIFY",)) + assert _verdict("true", enforcing=False) == ProviderDoesNotEnforceFips() + + +def test_provider_probe_agrees_with_whether_md5_is_usable_for_security_here(): + try: + hashlib.md5(b"", usedforsecurity=True) + except ValueError: + assert openssl_enforces_fips() is True + else: + assert openssl_enforces_fips() is False diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 89fd9c5c9d4..42ad00d6b13 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1681,6 +1681,110 @@ async def test_proxy_startup_refuses_an_unsafe_master_key_even_when_the_database assert ("could not be checked" in announced[0]) == key_can_have_encrypted_the_database +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_this_python_does_not_enforce_fips(monkeypatch, tmp_path): + from fastapi import FastAPI + + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + _, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: False) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError): + async with proxy_startup_event(FastAPI()): + pass + + assert len(announced) == 1 + assert "does not enforce FIPS" in announced[0] + + +@pytest.mark.asyncio +async def test_proxy_startup_refuses_fips_mode_when_the_config_disables_tls_verification(monkeypatch, tmp_path): + import yaml + from fastapi import FastAPI + + from litellm.proxy.common_utils.fips import FipsModeError + from litellm.proxy.proxy_server import proxy_startup_event + + config_path, announced = _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + config_path.write_text( + yaml.dump({"general_settings": {"master_key": "sk-a-safe-master-key"}, "litellm_settings": {"ssl_verify": False}}) + ) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError): + async with proxy_startup_event(FastAPI()): + pass + + assert "TLS certificate verification is disabled by litellm_settings.ssl_verify" in announced[0] + + +class _PrismaClientWhoseUserTableCannotHash: + class _Table: + async def find_many(self, where): + raise ValueError("[digital envelope routines] unsupported") + + class _Db: + litellm_usertable = None + + def __init__(self): + self.db = self._Db() + self.db.litellm_usertable = self._Table() + self.writer_db = self.db + + async def connect(self): + pass + + async def disconnect(self): + pass + + async def check_view_exists(self): + pass + + async def health_check(self): + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fips_mode", ["true", "false"]) +async def test_proxy_startup_surfaces_a_password_migration_crypto_failure(monkeypatch, tmp_path, caplog, fips_mode): + from fastapi import FastAPI + + from litellm.proxy.proxy_server import proxy_startup_event + + _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") + cannot_hash = _PrismaClientWhoseUserTableCannotHash() + + async def connected(**kwargs): + return cannot_hash + + monkeypatch.setattr("litellm.proxy.proxy_server.ProxyStartupEvent._setup_prisma_client", connected) + monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) + monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + if fips_mode == "true": + with pytest.raises(ValueError, match="digital envelope routines"): + async with proxy_startup_event(FastAPI()): + pass + else: + async with proxy_startup_event(FastAPI()): + await asyncio.sleep(0) + + failures = [r.getMessage() for r in caplog.records if "Password migration failed" in r.getMessage()] + assert len(failures) == 1 + assert "plaintext passwords stay unhashed" in failures[0] + assert "digital envelope routines" in failures[0] + + class _DatabaseWithOneStoredCredential: def __init__(self, ciphertext): self._ciphertext = ciphertext From 5c237840f4055a5a0058d09598ec51a66fae8e27 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:31:01 +0000 Subject: [PATCH 02/17] refactor(proxy): match ssl_verify off detection to runtime str_to_bool semantics Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 4 +++- tests/integration/_support/process.py | 15 ++++++++++----- .../test_litellm/proxy/common_utils/test_fips.py | 6 +++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 33f35388567..8a3c26b3220 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -6,6 +6,8 @@ from typing import Final from typing_extensions import assert_never +from litellm.secret_managers.main import str_to_bool + FIPS_MODE_ENV_VAR: Final = "LITELLM_FIPS_MODE" SSL_VERIFY_ENV_VAR: Final = "SSL_VERIFY" SSL_VERIFY_SETTING: Final = "litellm_settings.ssl_verify" @@ -146,6 +148,6 @@ def _is_off(value: object) -> bool: case bool(): return value is False case str(): - return value.strip().lower() in _FALSE_VALUES - {""} + return str_to_bool(value) is False case _: return False diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 6dec960097b..f65ab0e8093 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -140,6 +140,13 @@ def launched_proxy( assert root_stopped and not remaining, "Owned proxy required forced cleanup" +def _is_ready(client: httpx.Client) -> bool: + try: + return client.get("/health/readiness", timeout=2).status_code == 200 + except httpx.TransportError: + return False + + def refused_boot_log( gateway: Gateway, directory: Path, @@ -152,11 +159,9 @@ def refused_boot_log( with httpx.Client(base_url=f"http://127.0.0.1:{launched.port}", timeout=15, trust_env=False) as client: deadline: Final = time.monotonic() + 70 while launched.process.poll() is None: - try: - ready: Final = client.get("/health/readiness", timeout=2).status_code == 200 - except httpx.TransportError: - ready = False - assert not ready, f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + assert not _is_ready(client), ( + f"Proxy became ready instead of refusing to boot:\n{launched.log.read_text()}" + ) assert time.monotonic() < deadline, "Proxy neither exited nor became ready within the deadline" time.sleep(0.1) assert launched.process.returncode != 0, ( diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py index e934cafbdda..4b50ae4ad20 100644 --- a/tests/test_litellm/proxy/common_utils/test_fips.py +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -85,10 +85,10 @@ def test_on_with_a_non_enforcing_provider_is_refused_and_names_the_fix(): "ssl_env, ssl_setting, sources", [ ("false", True, ("SSL_VERIFY",)), - ("0", True, ("SSL_VERIFY",)), + (" FALSE ", True, ("SSL_VERIFY",)), (None, False, ("litellm_settings.ssl_verify",)), (None, "False", ("litellm_settings.ssl_verify",)), - ("no", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), + ("false", False, ("SSL_VERIFY", "litellm_settings.ssl_verify")), ], ) def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_setting, sources): @@ -99,7 +99,7 @@ def test_disabled_tls_verification_is_refused_naming_every_source(ssl_env, ssl_s assert "TLS certificate verification is disabled by " + " and ".join(sources) in str(refused.value) -@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, ""]) +@pytest.mark.parametrize("ssl_setting", [True, "true", "/etc/ssl/certs/ca.pem", None, "", "0", "no"]) def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_setting): assert _verdict("true", enforcing=True, ssl_setting=ssl_setting) == FipsModeOn() From 6ae5df66c1d24ba5ac460c2338ea01585753b04a Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 23 Sep 2026 07:45:39 +0000 Subject: [PATCH 03/17] refactor(proxy): drop tautological fips probe test and satisfy CodeQL return checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/fips.py | 23 ++++++++----------- .../proxy/common_utils/test_fips.py | 12 ---------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/common_utils/fips.py b/litellm/proxy/common_utils/fips.py index 8a3c26b3220..07bd4d2462c 100644 --- a/litellm/proxy/common_utils/fips.py +++ b/litellm/proxy/common_utils/fips.py @@ -121,9 +121,9 @@ def enforce_fips_boot_verdict(verdict: FipsBootVerdict, announce: Callable[[str] def render_refusal(refusal: FipsBootRefusal) -> str: match refusal: - case MalformedFipsMode(value=value): + case MalformedFipsMode(): return ( - f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={value} is not a boolean.\n" + f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR}={refusal.value} is not a boolean.\n" f"Set {FIPS_MODE_ENV_VAR} to true or false, or unset it." ) case ProviderDoesNotEnforceFips(): @@ -133,21 +133,18 @@ def render_refusal(refusal: FipsBootRefusal) -> str: "protected with algorithms the FIPS 140-3 policy forbids. Run the proxy from a FIPS image whose\n" f"OpenSSL FIPS provider is enabled, or unset {FIPS_MODE_ENV_VAR} on a non-FIPS runtime." ) - case TlsVerificationDisabled(sources=sources): + case TlsVerificationDisabled(): return ( f"{REFUSAL_PREFIX}: {FIPS_MODE_ENV_VAR} is on but TLS certificate verification is disabled by " - f"{' and '.join(sources)}.\nFIPS deployments must verify upstream certificates, so remove the " + f"{' and '.join(refusal.sources)}.\nFIPS deployments must verify upstream certificates, so remove the " "override or point ssl_verify at a CA bundle instead." ) - case _: - assert_never(refusal) + return assert_never(refusal) def _is_off(value: object) -> bool: - match value: - case bool(): - return value is False - case str(): - return str_to_bool(value) is False - case _: - return False + if isinstance(value, bool): + return value is False + if isinstance(value, str): + return str_to_bool(value) is False + return False diff --git a/tests/test_litellm/proxy/common_utils/test_fips.py b/tests/test_litellm/proxy/common_utils/test_fips.py index 4b50ae4ad20..0e8138856c0 100644 --- a/tests/test_litellm/proxy/common_utils/test_fips.py +++ b/tests/test_litellm/proxy/common_utils/test_fips.py @@ -1,5 +1,3 @@ -import hashlib - import pytest from litellm.proxy.common_utils.fips import ( @@ -12,7 +10,6 @@ from litellm.proxy.common_utils.fips import ( enforce_fips_boot_verdict, fips_boot_verdict, is_fips_mode, - openssl_enforces_fips, parse_fips_mode, ) @@ -107,12 +104,3 @@ def test_verified_or_custom_bundle_tls_settings_are_not_treated_as_disabled(ssl_ def test_disabled_tls_is_reported_before_the_provider_so_operators_see_config_mistakes_first(): assert _verdict("true", enforcing=False, ssl_env="false") == TlsVerificationDisabled(sources=("SSL_VERIFY",)) assert _verdict("true", enforcing=False) == ProviderDoesNotEnforceFips() - - -def test_provider_probe_agrees_with_whether_md5_is_usable_for_security_here(): - try: - hashlib.md5(b"", usedforsecurity=True) - except ValueError: - assert openssl_enforces_fips() is True - else: - assert openssl_enforces_fips() is False From e6a5d32cd9925caedbd1eb067b86772031dcf556 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 06:07:49 +0000 Subject: [PATCH 04/17] test(proxy): inject the fake Prisma client through the module boundary instead of patching _setup_prisma_client Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/proxy/test_proxy_server.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 42ad00d6b13..39e4a417d19 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -1733,7 +1733,7 @@ class _PrismaClientWhoseUserTableCannotHash: class _Db: litellm_usertable = None - def __init__(self): + def __init__(self, database_url, proxy_logging_obj): self.db = self._Db() self.db.litellm_usertable = self._Table() self.writer_db = self.db @@ -1744,6 +1744,9 @@ class _PrismaClientWhoseUserTableCannotHash: async def disconnect(self): pass + def start_view_setup_task(self): + pass + async def check_view_exists(self): pass @@ -1761,12 +1764,7 @@ async def test_proxy_startup_surfaces_a_password_migration_crypto_failure(monkey _boot_with_general_settings(monkeypatch, tmp_path, {"master_key": "sk-a-safe-master-key"}) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setenv("DATABASE_URL", "postgresql://nobody:nothing@127.0.0.1:1/unreachable") - cannot_hash = _PrismaClientWhoseUserTableCannotHash() - - async def connected(**kwargs): - return cannot_hash - - monkeypatch.setattr("litellm.proxy.proxy_server.ProxyStartupEvent._setup_prisma_client", connected) + monkeypatch.setattr("litellm.proxy.proxy_server.PrismaClient", _PrismaClientWhoseUserTableCannotHash) monkeypatch.setattr("litellm.proxy.proxy_server.openssl_enforces_fips", lambda: True) monkeypatch.setenv("LITELLM_FIPS_MODE", fips_mode) From e8cee19bee51ac472d6cbb85a90e5bb8c66d5406 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:32:05 +0000 Subject: [PATCH 05/17] feat(proxy): default at-rest encryption to AES-256-GCM with HKDF v3 and make PyNaCl optional New writes use v3:gcm: (AES-256-GCM under HKDF-SHA256). v2:gcm: and unprefixed XSalsa20 values stay readable under the raw SHA-256 derivation they were written with. xsalsa20-poly1305 remains an explicit opt-in and is refused under LITELLM_FIPS_MODE. PyNaCl imports are lazy and a missing PyNaCl on legacy ciphertext raises one error naming the legacy-encryption extra and the re-encrypt path. pynacl moves from the proxy extra to legacy-encryption, which the standard Dockerfiles install Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- Dockerfile | 2 + backend/Dockerfile | 2 + docker/Dockerfile.database | 2 + docker/Dockerfile.non_root | 3 + gateway/Dockerfile | 2 + .../mcp_server/bridge_token_flow.py | 6 +- litellm/proxy/_experimental/mcp_server/db.py | 2 +- .../outbound_credentials/envelope.py | 26 +- .../proxy/client/cli/commands/encryption.py | 8 +- .../common_utils/encrypt_decrypt_utils.py | 172 ++++++----- .../credential_migration.py | 41 ++- .../key_management_endpoints.py | 4 +- migrations/Dockerfile | 2 + pyproject.toml | 3 +- .../test_at_rest_encryption_format.py | 263 +++++++++++++++++ .../outbound_credentials/test_envelope.py | 5 +- .../mcp_server/test_db_credentials.py | 2 +- .../test_encrypt_decrypt_utils.py | 276 ++++++++++++------ .../test_credential_migration.py | 32 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- uv.lock | 8 +- 21 files changed, 619 insertions(+), 246 deletions(-) create mode 100644 tests/integration/management/test_at_rest_encryption_format.py diff --git a/Dockerfile b/Dockerfile index 4dcecf3ea3d..332fc1f40c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ # Install third-party dependencies (cached unless pyproject.toml/uv.lock change) RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ @@ -100,6 +101,7 @@ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui. # Install project and workspace packages (fast - deps already cached) RUN uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ diff --git a/backend/Dockerfile b/backend/Dockerfile index 59f836b55f8..550296dca0b 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -43,6 +43,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ @@ -55,6 +56,7 @@ COPY . . RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 61b6faae691..b71cc445150 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -76,6 +76,7 @@ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ # Install third-party dependencies (cached unless pyproject.toml/uv.lock change) RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ @@ -98,6 +99,7 @@ RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui. # Install project and workspace packages (fast - deps already cached) RUN uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index d4c07d56d90..364a763c7fa 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -82,6 +82,7 @@ COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ @@ -110,6 +111,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \ uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ @@ -120,6 +122,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ else \ uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 8045a8b64cb..e0a5ad4081e 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -59,6 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ @@ -71,6 +72,7 @@ COPY . . RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra proxy-runtime \ --extra extra_proxy \ --extra semantic-router \ diff --git a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py index e5d11271c67..27ed7ce3178 100644 --- a/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py +++ b/litellm/proxy/_experimental/mcp_server/bridge_token_flow.py @@ -14,9 +14,7 @@ from typing_extensions import assert_never from litellm._logging import verbose_logger from litellm.proxy._experimental.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS -from litellm.proxy.common_utils.encrypt_decrypt_utils import ( - _V2_GCM_PREFIX, # pyright: ignore[reportPrivateUsage] # reuse the encrypted credential's format discriminator -) +from litellm.proxy.common_utils.encrypt_decrypt_utils import is_versioned_gcm from litellm.types.mcp_server.mcp_server_manager import MCPServer if TYPE_CHECKING: @@ -99,7 +97,7 @@ async def _opaque_bearer_is_gateway_credential(token: str) -> bool: user_api_key_cache, ) - if is_envelope(token) or is_refresh_envelope(token) or token.startswith(_V2_GCM_PREFIX): + if is_envelope(token) or is_refresh_envelope(token) or is_versioned_gcm(token): return True try: if ExperimentalUIJWTToken.get_key_object_from_ui_hash_key(token) is not None: diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 70c6e6f4bf3..1ec0f24ae38 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1418,7 +1418,7 @@ async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient, def _decode_user_credential(stored: str) -> str | None: """Read back a value persisted in ``LiteLLM_MCPUserCredentials.credential_b64``. - Tries nacl decryption first (current write format). Falls back to a + Tries the at-rest decryptor first (current write format). Falls back to a plain ``urlsafe_b64decode`` for rows persisted by older code that wrote the credential without encryption. Returns ``None`` when neither path yields a valid string. diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py index df883d5a208..67962b589cf 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/envelope.py @@ -33,14 +33,13 @@ defend against non-UTF-8 field content that cannot survive JSON parsing. from __future__ import annotations -import base64 from datetime import datetime, timedelta from typing import Final, Literal, TypeAlias import jwt from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value, encrypt_value +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with, encrypt_value_helper ENVELOPE_PREFIX: Final = "llm_env_" """Marker prefix on every serialized ACCESS envelope so the edge can cheaply tell an envelope @@ -540,22 +539,15 @@ def _decode_claims( def _encrypt_grant_blob(plaintext: str, encryption_key: SecretStr) -> str: - ciphertext: Final = bytes(encrypt_value(value=plaintext, signing_key=encryption_key.get_secret_value())) - return base64.urlsafe_b64encode(ciphertext).decode("ascii") + return encrypt_value_helper(value=plaintext, new_encryption_key=encryption_key.get_secret_value()) def _decrypt_grant( blob: str, encryption_key: SecretStr, ) -> UpstreamTokenGrant | DecryptFailed | MalformedPayload: - from nacl.exceptions import CryptoError - - try: - plaintext: Final = decrypt_value( - value=base64.urlsafe_b64decode(blob), - signing_key=encryption_key.get_secret_value(), - ) - except (CryptoError, ValueError): + plaintext: Final = decrypt_if_encrypted_with(value=blob, signing_key=encryption_key.get_secret_value()) + if plaintext is None: return DecryptFailed() try: return UpstreamTokenGrant.model_validate_json(plaintext) @@ -567,14 +559,8 @@ def _decrypt_refresh( blob: str, encryption_key: SecretStr, ) -> RefreshCredential | DecryptFailed | MalformedPayload: - from nacl.exceptions import CryptoError - - try: - plaintext: Final = decrypt_value( - value=base64.urlsafe_b64decode(blob), - signing_key=encryption_key.get_secret_value(), - ) - except (CryptoError, ValueError): + plaintext: Final = decrypt_if_encrypted_with(value=blob, signing_key=encryption_key.get_secret_value()) + if plaintext is None: return DecryptFailed() try: return RefreshCredential.model_validate_json(plaintext) diff --git a/litellm/proxy/client/cli/commands/encryption.py b/litellm/proxy/client/cli/commands/encryption.py index f9a9356d0d6..9d332bee442 100644 --- a/litellm/proxy/client/cli/commands/encryption.py +++ b/litellm/proxy/client/cli/commands/encryption.py @@ -29,11 +29,11 @@ def encryption(): ) @click.pass_context def migrate(ctx: click.Context, check_only: bool, dry_run: bool): - """Re-encrypt at-rest credentials into the AES-256-GCM (v2:gcm:) format. + """Re-encrypt at-rest credentials into the versioned AES-256-GCM (v3:gcm:) format. - Requires the proxy to be started with - ``general_settings.encryption_algorithm: aes-256-gcm``. Idempotent and - resumable; safe to re-run after an interruption. + Requires the proxy to write ``aes-256-gcm`` (the default; refused while + ``general_settings.encryption_algorithm: xsalsa20-poly1305`` is set). Idempotent + and resumable; safe to re-run after an interruption. Examples: lite encryption migrate --check # attestation scan, no writes diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 3584aaaf833..9f91644abb2 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -1,27 +1,49 @@ import base64 +import hashlib import os from collections.abc import Mapping -from typing import Final, Literal, cast +from typing import TYPE_CHECKING, Final, Literal, cast from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.proxy.common_utils.fips import FipsModeError, is_fips_mode -# Versioned ciphertext marker for AES-256-GCM values. -# Format: "v2:gcm:" + base64url(nonce(12) || ciphertext || tag(16)). +if TYPE_CHECKING: + from nacl.secret import SecretBox + +# Versioned ciphertext markers for AES-256-GCM values, each +# ":gcm:" + base64url(nonce(12) || ciphertext || tag(16)). +# v3 derives the key with HKDF-SHA256, v2 with a raw SHA-256 of the salt key. # Legacy XSalsa20-Poly1305 (nacl) values carry no marker; the colon in the # prefix can never appear in base64url(nacl output), so the prefix check is an -# unambiguous discriminator between the two formats on read. +# unambiguous discriminator between the formats on read. +_V3_GCM_PREFIX: Final = "v3:gcm:" _V2_GCM_PREFIX: Final = "v2:gcm:" +_GCM_PREFIXES: Final = (_V3_GCM_PREFIX, _V2_GCM_PREFIX) +_HKDF_INFO: Final = b"litellm-at-rest-v3" # general_settings key selecting the at-rest encryption algorithm for new writes. -# Default preserves the legacy algorithm so existing deployments are byte-for-byte -# unchanged until they explicitly opt in. Decrypt is always format-detecting, so -# flipping this flag forward (or back) never strands previously-written data. +# Decrypt is always format-detecting, so the setting never strands stored data. _ENCRYPTION_ALGORITHM_SETTING: Final = "encryption_algorithm" _ALGO_AES_GCM: Final = "aes-256-gcm" _ALGO_XSALSA20: Final = "xsalsa20-poly1305" +_NACL_MIN_CIPHERTEXT_BYTES: Final = 40 +_LEGACY_ENCRYPTION_HELP: Final = ( + "Install the legacy-encryption extra (pip install 'litellm[legacy-encryption]') on a non-FIPS image and " + "re-encrypt stored credentials with `litellm-proxy encryption migrate` (POST /credentials/migrate-encryption) " + "before running without PyNaCl" +) + + +class LegacyEncryptionUnavailableError(RuntimeError): + pass + + +def is_versioned_gcm(value: str) -> bool: + return value.startswith(_GCM_PREFIXES) + def _get_salt_key(): from litellm.proxy.proxy_server import master_key @@ -35,63 +57,66 @@ def _get_salt_key(): def _get_encryption_algorithm() -> str: - """ - Resolve the configured at-rest encryption algorithm for *new writes*. + """Resolve the at-rest encryption algorithm for new writes from ``general_settings.encryption_algorithm``. - Read from ``general_settings.encryption_algorithm`` at write time. Defaults to - the legacy XSalsa20-Poly1305 algorithm so deployments that have not opted in - keep producing byte-for-byte identical ciphertext. + Defaults to AES-256-GCM. ``xsalsa20-poly1305`` stays available as an explicit opt-in for deployments that + still need byte-for-byte legacy output, except under ``LITELLM_FIPS_MODE`` where it is refused. """ try: from litellm.proxy.proxy_server import general_settings - algo: Final = general_settings.get(_ENCRYPTION_ALGORITHM_SETTING, _ALGO_XSALSA20) - except Exception: - # general_settings may not be importable in some contexts (e.g. SDK-only - # use of these helpers). Fall back to the legacy algorithm. - return _ALGO_XSALSA20 - - if isinstance(algo, str) and algo.lower() == _ALGO_AES_GCM: + algo: Final = general_settings.get(_ENCRYPTION_ALGORITHM_SETTING, _ALGO_AES_GCM) + except Exception: # noqa: BLE001 # proxy_server is not importable in SDK-only use of these helpers return _ALGO_AES_GCM + + if not isinstance(algo, str) or algo.lower() != _ALGO_XSALSA20: + return _ALGO_AES_GCM + if is_fips_mode(): + raise FipsModeError( + f"general_settings.{_ENCRYPTION_ALGORITHM_SETTING}={_ALGO_XSALSA20} is not allowed under " + f"LITELLM_FIPS_MODE: XSalsa20-Poly1305 is not a FIPS approved algorithm. Remove the setting to write " + f"{_ALGO_AES_GCM}" + ) return _ALGO_XSALSA20 -def _derive_key(signing_key: str) -> bytes: - """Derive a 32-byte key from the salt/master key (shared by both algorithms). - - Known limitation: this is a single-pass, unsalted ``SHA-256`` of the key, not - a dedicated KDF (HKDF/PBKDF2). It is the *same* derivation the legacy nacl - path already uses, so the AES path introduces no new weakness and stays - interoperable with existing key sourcing; AES-256-GCM's per-value 12-byte - random nonce gives the unique (key, nonce) pairs GCM requires. Moving both - algorithms to HKDF-SHA256 would be more defensible in an audit but is a - separate, coordinated change (it must re-derive or re-encrypt existing data). - """ - import hashlib - +def _derive_key_sha256(signing_key: str) -> bytes: + """Historical derivation shared by legacy nacl values and ``v2:gcm:`` values: one unsalted SHA-256.""" return hashlib.sha256(signing_key.encode()).digest() +def _derive_key_hkdf(signing_key: str) -> bytes: + """Derivation for ``v3:gcm:`` values: HKDF-SHA256 with a fixed info string.""" + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + return HKDF(algorithm=hashes.SHA256(), length=32, salt=None, info=_HKDF_INFO).derive(signing_key.encode()) + + def _encrypt_aes_gcm(value: str, signing_key: str) -> str: - """Encrypt under AES-256-GCM and return the versioned ``v2:gcm:`` string.""" + """Encrypt under AES-256-GCM and return the versioned ``v3:gcm:`` string.""" from cryptography.hazmat.primitives.ciphers.aead import AESGCM nonce: Final = os.urandom(12) # AESGCM.encrypt returns ciphertext || tag(16); wire format is nonce || that. - blob: Final = AESGCM(_derive_key(signing_key)).encrypt(nonce, value.encode("utf-8"), None) - return _V2_GCM_PREFIX + base64.urlsafe_b64encode(nonce + blob).decode("utf-8") + blob: Final = AESGCM(_derive_key_hkdf(signing_key)).encrypt(nonce, value.encode("utf-8"), None) + return _V3_GCM_PREFIX + base64.urlsafe_b64encode(nonce + blob).decode("utf-8") def _decrypt_aes_gcm(value: str, signing_key: str) -> str: - """Decrypt a versioned ``v2:gcm:`` string produced by :func:`_encrypt_aes_gcm`.""" + """Decrypt a versioned ``v3:gcm:`` or ``v2:gcm:`` string, deriving the key the way its version was written.""" from cryptography.hazmat.primitives.ciphers.aead import AESGCM - raw: Final = base64.urlsafe_b64decode(value[len(_V2_GCM_PREFIX) :]) + if value.startswith(_V3_GCM_PREFIX): + key, prefix = _derive_key_hkdf(signing_key), _V3_GCM_PREFIX + else: + key, prefix = _derive_key_sha256(signing_key), _V2_GCM_PREFIX + raw: Final = base64.urlsafe_b64decode(value[len(prefix) :]) # An empty plaintext still serializes to nonce(12) || tag(16) = 28 bytes, so a # short/empty buffer here is a corrupt value: let AESGCM.decrypt raise and be # swallowed by decrypt_value_helper (returns None/original), same as legacy. nonce, blob = raw[:12], raw[12:] - return AESGCM(_derive_key(signing_key)).decrypt(nonce, blob, None).decode("utf-8") + return AESGCM(key).decrypt(nonce, blob, None).decode("utf-8") def encrypt_value_helper(value: str, new_encryption_key: str | None = None): @@ -100,7 +125,7 @@ def encrypt_value_helper(value: str, new_encryption_key: str | None = None): try: if isinstance(value, str): if _get_encryption_algorithm() == _ALGO_AES_GCM: - # AES path: the v2:gcm: output is already a base64url string, so it + # AES path: the v3:gcm: output is already a base64url string, so it # is returned directly with no extra base64 wrapper. return _encrypt_aes_gcm(value=value, signing_key=cast(str, signing_key)) @@ -131,19 +156,25 @@ def _legacy_ciphertext_bytes(value: str) -> bytes: def _decrypt_with_signing_key(value: str, signing_key: str) -> str: # Versioned AES-256-GCM values are detected before any base64 decode. # The prefix is the algorithm tag the legacy nacl format never carried. - if value.startswith(_V2_GCM_PREFIX): + if is_versioned_gcm(value): return _decrypt_aes_gcm(value=value, signing_key=signing_key) return decrypt_value(value=_legacy_ciphertext_bytes(value), signing_key=signing_key) def decrypt_if_encrypted_with(value: str, signing_key: str) -> str | None: - """None unless value is a ciphertext under signing_key.""" + """None unless value is a ciphertext under signing_key. + + A legacy ciphertext met without PyNaCl installed is not silently "not a ciphertext": it raises + :class:`LegacyEncryptionUnavailableError` naming the re-encrypt path. + """ try: # base64 decoding skips characters outside its alphabet, so "" and "*" decode to no bytes, # which decrypt_value reads as an empty plaintext under any key. - decodes_to_nothing: Final = not value.startswith(_V2_GCM_PREFIX) and not _legacy_ciphertext_bytes(value) + decodes_to_nothing: Final = not is_versioned_gcm(value) and not _legacy_ciphertext_bytes(value) return None if decodes_to_nothing else _decrypt_with_signing_key(value=value, signing_key=signing_key) + except LegacyEncryptionUnavailableError: + raise except Exception: # noqa: BLE001 # base64, nacl and AES-GCM each raise their own "not a ciphertext" type return None @@ -162,6 +193,8 @@ def decrypt_value_helper( # if it's not str - do not decrypt it, return the value return value + except LegacyEncryptionUnavailableError: + raise except Exception as e: error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {e}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" if exception_type == "debug": @@ -177,50 +210,27 @@ def decrypt_value_helper( return None -def encrypt_value(value: str, signing_key: str): - import hashlib +def _legacy_secret_box(signing_key: str, purpose: str) -> "SecretBox": + try: + import nacl.secret + except ImportError as error: + raise LegacyEncryptionUnavailableError( + f"Cannot {purpose} with the legacy {_ALGO_XSALSA20} algorithm: PyNaCl is not installed. " + f"{_LEGACY_ENCRYPTION_HELP}" + ) from error + return nacl.secret.SecretBox(_derive_key_sha256(signing_key)) - import nacl.secret - import nacl.utils - # get 32 byte master key # - hash_object: Final = hashlib.sha256(signing_key.encode()) - hash_bytes: Final = hash_object.digest() - - # initialize secret box # - box: Final = nacl.secret.SecretBox(hash_bytes) - - # encode message # - value_bytes: Final = value.encode("utf-8") - - encrypted: Final = box.encrypt(value_bytes) - - return encrypted +def encrypt_value(value: str, signing_key: str) -> bytes: + return bytes(_legacy_secret_box(signing_key, "encrypt").encrypt(value.encode("utf-8"))) def decrypt_value(value: bytes, signing_key: str) -> str: - import hashlib - - import nacl.secret - import nacl.utils - - # get 32 byte master key # - hash_object: Final = hashlib.sha256(signing_key.encode()) - hash_bytes: Final = hash_object.digest() - - # initialize secret box # - box: Final = nacl.secret.SecretBox(hash_bytes) - - # Convert the bytes object to a string - try: - if len(value) == 0: - return "" - - plaintext = box.decrypt(value) - plaintext = plaintext.decode("utf-8") - return plaintext - except Exception as e: - raise e + if len(value) == 0: + return "" + if len(value) < _NACL_MIN_CIPHERTEXT_BYTES: + raise ValueError(f"Value of {len(value)} bytes is too short to be a {_ALGO_XSALSA20} ciphertext") + return _legacy_secret_box(signing_key, "decrypt a stored value").decrypt(value).decode("utf-8") class SecretMapDecodeError(RuntimeError): diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index 915cce87dbd..181a0e2b016 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -2,9 +2,9 @@ At-rest credential re-encryption migration. Switches every encrypted-at-rest value from the legacy XSalsa20-Poly1305 (nacl) -format to the versioned AES-256-GCM (``v2:gcm:``) format produced by -``encrypt_decrypt_utils`` when ``general_settings.encryption_algorithm`` is set to -``aes-256-gcm``. +format to the versioned AES-256-GCM (``v3:gcm:``) format produced by +``encrypt_decrypt_utils`` by default (``general_settings.encryption_algorithm`` +not set to the legacy ``xsalsa20-poly1305``). Design properties (see case 2026-06-24 fix plan): @@ -12,7 +12,7 @@ Design properties (see case 2026-06-24 fix plan): it re-encrypts existing ciphertext under the same derived key but in the new AES format. This is achieved by decrypting with the format-detecting reader and re-encrypting through ``encrypt_value_helper`` with the AES gate enabled. -* **Idempotent.** A value already carrying the ``v2:gcm:`` prefix is recognised +* **Idempotent.** A value already carrying a ``v3:gcm:`` or ``v2:gcm:`` prefix is recognised and left untouched, so re-running the migration is a no-op on migrated rows. * **Resumable.** Walkers commit per row (or per small table), so an interrupted run leaves a clean mixed state that a re-run completes. @@ -42,12 +42,13 @@ if TYPE_CHECKING: from litellm.proxy.common_utils.encrypt_decrypt_utils import ( _ALGO_AES_GCM, _ENCRYPTION_ALGORITHM_SETTING, - _V2_GCM_PREFIX, SecretMapDecodeError, + _get_encryption_algorithm, _get_salt_key, decode_secret_map, decrypt_value_helper, encrypt_value_helper, + is_versioned_gcm, ) ValueClass = Literal["migrated", "legacy", "plaintext", "undecryptable", "not-a-string"] @@ -59,7 +60,7 @@ class LocationReport: location: str scanned: int = 0 - migrated: int = 0 # values rewritten to v2 this run + migrated: int = 0 # values rewritten to versioned AES this run already_v2: int = 0 # values already migrated (skipped) plaintext: int = 0 # legacy-plaintext values (no ciphertext to migrate) undecryptable: int = 0 # could not decrypt — preserved, not overwritten @@ -124,15 +125,15 @@ class MigrationReport: def is_migrated(value: object) -> bool: - """True if ``value`` is already an AES-256-GCM (``v2:gcm:``) ciphertext.""" - return isinstance(value, str) and value.startswith(_V2_GCM_PREFIX) + """True if ``value`` is already a versioned AES-256-GCM (``v3:gcm:`` or ``v2:gcm:``) ciphertext.""" + return isinstance(value, str) and is_versioned_gcm(value) def classify_value(value: object, key: str = "scan") -> ValueClass: """Classify a stored value for the residual scanner. * ``not-a-string`` — not a string (numbers/bools/None left as-is on disk). - * ``migrated`` — carries the ``v2:gcm:`` prefix. + * ``migrated`` — carries a versioned ``gcm`` prefix. * ``legacy`` — decrypts under the legacy nacl reader (still needs migrating). * ``plaintext`` — a non-empty string that does not decrypt and is not v2; treated as legacy plaintext (nothing to migrate). @@ -144,7 +145,7 @@ def classify_value(value: object, key: str = "scan") -> ValueClass: return "not-a-string" if value == "": return "plaintext" - if value.startswith(_V2_GCM_PREFIX): + if is_versioned_gcm(value): return "migrated" decrypted: Final = decrypt_value_helper(value=value, key=key, exception_type="debug", return_original_value=False) if decrypted is None: @@ -156,14 +157,14 @@ def classify_value(value: object, key: str = "scan") -> ValueClass: def reencrypt_value(value: object, key: str = "migrate") -> object: """Re-encrypt a single stored string into the configured (AES) format. - Returns the value unchanged if it is not a string, is already ``v2:``, or + Returns the value unchanged if it is not a string, is already versioned AES, or cannot be decrypted (skip-on-undecryptable). Otherwise decrypts under the format-detecting reader and re-encrypts through ``encrypt_value_helper`` (which writes AES when the gate is on). """ if not isinstance(value, str) or value == "": return value - if value.startswith(_V2_GCM_PREFIX): + if is_versioned_gcm(value): return value # idempotent: already migrated decrypted: Final = decrypt_value_helper(value=value, key=key, exception_type="debug", return_original_value=False) if decrypted is None: @@ -194,13 +195,11 @@ def _assert_aes_gate_enabled() -> None: Running the migration with the gate off would decrypt then re-encrypt right back into the legacy format — a no-op that silently fails the migration. """ - from litellm.proxy.proxy_server import general_settings - - algo: Final = general_settings.get(_ENCRYPTION_ALGORITHM_SETTING) - if not (isinstance(algo, str) and algo.lower() == _ALGO_AES_GCM): + algo: Final = _get_encryption_algorithm() + if algo != _ALGO_AES_GCM: raise RuntimeError( - "Encryption migration requires general_settings.encryption_algorithm: " - f"'{_ALGO_AES_GCM}'. Current value: {algo!r}. Set it before migrating " + f"Encryption migration requires general_settings.{_ENCRYPTION_ALGORITHM_SETTING}: " + f"'{_ALGO_AES_GCM}' (the default). Current value: {algo!r}. Remove the legacy opt-in before migrating " "so re-encrypted values are written in the AES-256-GCM format." ) @@ -427,7 +426,7 @@ def _classify_callback_value(value: object) -> ValueClass: Encrypted callback vars carry the ``litellm_enc::`` marker in front of the ciphertext; strip it, then classify the inner value the same way the - covered-table scanner does (``v2:gcm:`` prefix -> migrated, nacl-decryptable + covered-table scanner does (versioned ``gcm`` prefix -> migrated, nacl-decryptable -> legacy, otherwise plaintext). Detecting legacy by decrypt rather than by a re-encrypt delta is what makes the ``check_encryption`` attestation correct even when run with the AES write gate off. @@ -625,8 +624,8 @@ async def migrate_encryption( ) -> MigrationReport: """Run the full at-rest re-encryption migration. - Requires ``general_settings.encryption_algorithm == 'aes-256-gcm'`` so writes - are produced in the AES format. Idempotent and resumable: re-running skips + Requires the proxy to write ``aes-256-gcm`` (the default) so re-encrypted + values are produced in the AES format. Idempotent and resumable: re-running skips already-migrated values and finishes any partial run. A ``dry_run`` performs no writes: the covered tables are scanned read-only diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 306ea90d7f1..e4d3b7dbffc 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -5332,9 +5332,9 @@ async def migrate_encryption_endpoint( ), ): """ - Re-encrypt all at-rest credentials into the AES-256-GCM (``v2:gcm:``) format. + Re-encrypt all at-rest credentials into the versioned AES-256-GCM (``v3:gcm:``) format. - Admin only. Requires ``general_settings.encryption_algorithm: aes-256-gcm``. + Admin only. Requires the proxy to write ``aes-256-gcm`` (the default). Idempotent and resumable — re-running skips already-migrated values. Pass ``dry_run=true`` for a non-mutating scan (equivalent to ``--check``). """ diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 255b94b0ea8..b2038b3b5a8 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -55,6 +55,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \ uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra extra_proxy \ --python python3.13 @@ -64,6 +65,7 @@ COPY . . RUN --mount=type=cache,target=/root/.cache/uv \ uv sync --frozen --no-default-groups --no-editable \ --extra proxy \ + --extra legacy-encryption \ --extra extra_proxy \ --python python3.13 diff --git a/pyproject.toml b/pyproject.toml index ba72378989a..d51aefaa607 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,6 @@ proxy = [ "PyJWT>=2.13.0,<3.0", "python-multipart>=0.0.27,<1.0", "cryptography>=49.0.0,<51.0", - "pynacl>=1.6.2,<2.0", "websockets>=15.0.1,<16.0", "boto3>=1.43.1,<2.0", "azure-identity>=1.25.2,<2.0", @@ -121,6 +120,8 @@ utils = [ "numpydoc>=1.8.0,<2.0", ] caching = ["diskcache>=5.6.3,<6.0"] +# Reads and writes the legacy XSalsa20-Poly1305 at-rest format; the FIPS image omits it +legacy-encryption = ["pynacl>=1.6.2,<2.0"] mcp = ["mcp>=2.2.0,<3", "httpx2>=2.5.0,<3", "pydantic>=2.12.0,<3"] # Driver for the MongoDB Atlas vector store; Atlas Vector Search has no HTTP query API. # The floor is 4.9 because that is the release AsyncMongoClient landed in. diff --git a/tests/integration/management/test_at_rest_encryption_format.py b/tests/integration/management/test_at_rest_encryption_format.py new file mode 100644 index 00000000000..6ddc07307de --- /dev/null +++ b/tests/integration/management/test_at_rest_encryption_format.py @@ -0,0 +1,263 @@ +"""At-rest encryption writes the versioned AES-256-GCM format and keeps reading every older format. + +The proxy, Postgres and the router run for real. Older rows are built by hand from the wire formats litellm +owns (nacl SecretBox and AES-GCM under the raw SHA-256 derivation) and inserted straight into the model table, +the way a deployment upgraded in place would find them. Every read is proven at the scripted upstream: the +Authorization header it receives is the plaintext key the proxy decrypted. +""" + +import base64 +import hashlib +import json +import os +import signal +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final + +import httpx +import nacl.secret +import psycopg +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy_process + +V3_PREFIX: Final = "v3:gcm:" +V2_PREFIX: Final = "v2:gcm:" + + +def _salt_key() -> str: + return os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt") + + +def _sha256_key() -> bytes: + return hashlib.sha256(_salt_key().encode()).digest() + + +def _legacy_nacl_ciphertext(plaintext: str) -> str: + sealed: Final = nacl.secret.SecretBox(_sha256_key()).encrypt(plaintext.encode()) + return base64.urlsafe_b64encode(bytes(sealed)).decode() + + +def _legacy_v2_ciphertext(plaintext: str) -> str: + nonce: Final = os.urandom(12) + blob: Final = AESGCM(_sha256_key()).encrypt(nonce, plaintext.encode(), None) + return V2_PREFIX + base64.urlsafe_b64encode(nonce + blob).decode() + + +def _stored_api_key(model_id: str) -> str: + rows: Final = read_rows( + "SELECT litellm_params->>'api_key' AS api_key FROM \"LiteLLM_ProxyModelTable\" WHERE model_id = %s", + (model_id,), + ) + assert len(rows) == 1, rows + return string_value(rows[0]["api_key"]) + + +def _stored_credential_api_key(name: str) -> str: + rows: Final = read_rows( + "SELECT credential_values->>'api_key' AS api_key FROM \"LiteLLM_CredentialsTable\" WHERE credential_name = %s", + (name,), + ) + assert len(rows) == 1, rows + return string_value(rows[0]["api_key"]) + + +def _insert_model_row(gateway: Gateway, scenario: Scenario, ciphertext: str) -> tuple[str, str]: + model_id: Final = f"enc-{uuid.uuid4().hex}" + model_name: Final = f"integration-{uuid.uuid4().hex}" + params: Final = { + "model": "openai/gpt-4o-mini", + "api_key": ciphertext, + "api_base": f"{gateway.upstream_url}/v1", + } + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection: + connection.execute( + 'INSERT INTO "LiteLLM_ProxyModelTable" ' + "(model_id, model_name, litellm_params, model_info, created_by, updated_by) " + "VALUES (%s, %s, %s::jsonb, %s::jsonb, %s, %s)", + (model_id, model_name, json.dumps(params), json.dumps({"id": model_id}), "integration", "integration"), + ) + scenario.cleanups.callback(_delete_model_row_if_present, gateway, model_id) + return model_id, model_name + + +def _delete_model_row_if_present(gateway: Gateway, model_id: str) -> None: + response: Final = gateway.request("POST", "/model/delete", {"id": model_id}) + assert response.status_code in (200, 400, 404), response.text + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True) as connection: + connection.execute('DELETE FROM "LiteLLM_ProxyModelTable" WHERE model_id = %s', (model_id,)) + + +def _new_model(gateway: Gateway, scenario: Scenario, api_key: str) -> tuple[str, str]: + model_name: Final = f"integration-{uuid.uuid4().hex}" + created: Final = gateway.post( + "/model/new", + { + "model_name": model_name, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": api_key, + "api_base": f"{gateway.upstream_url}/v1", + }, + "model_info": {}, + }, + ) + model_id: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, model_id) + return model_id, model_name + + +def _drain_upstream(gateway: Gateway) -> tuple[tuple[str, str], ...]: + with httpx.Client(base_url=gateway.upstream_url, timeout=15, trust_env=False) as upstream: + observed: Final = upstream.get("/__observations") + observed.raise_for_status() + requests: Final = JSON_OBJECT.validate_json(observed.content)["requests"] + assert isinstance(requests, list), requests + entries: Final = tuple(object_value(value) for value in requests) + return tuple((json.dumps(entry["body"]), string_value(entry["authorization"])) for entry in entries) + + +def _authorizations_for(observed: tuple[tuple[str, str], ...], marker: str) -> tuple[str, ...]: + return tuple(authorization for body, authorization in observed if marker in body) + + +def _chat_reaches_upstream_with(gateway: Gateway, model_name: str, plaintext_key: str) -> None: + marker: Final = f"marker-{uuid.uuid4().hex}" + response: Final = eventually( + lambda: gateway.request( + "POST", "/v1/chat/completions", {"model": model_name, "messages": [{"role": "user", "content": marker}]} + ), + lambda observed: observed.status_code == 200, + seconds=70, + ) + assert response.status_code == 200, response.text + assert _authorizations_for(_drain_upstream(gateway), marker) == (f"Bearer {plaintext_key}",) + + +def _model_table_count(gateway: Gateway, counter: str) -> int: + report: Final = object_value(gateway.get("/credentials/migrate-encryption/check")["report"]) + value: Final = object_value(object_value(report["locations"])["model_table"])[counter] + assert isinstance(value, int), report + return value + + +def test_new_model_api_key_is_stored_as_v3_gcm_and_decrypts_for_the_upstream_call(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + plaintext_key: Final = f"sk-upstream-{uuid.uuid4().hex}" + model_id, model_name = _new_model(gateway, scenario, plaintext_key) + stored: Final = _stored_api_key(model_id) + assert stored.startswith(V3_PREFIX), f"expected a {V3_PREFIX} ciphertext, stored {stored!r}" + assert plaintext_key not in stored + _chat_reaches_upstream_with(gateway, model_name, plaintext_key) + + +def test_new_credential_values_are_stored_as_v3_gcm(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + name: Final = f"credential-{uuid.uuid4().hex}" + plaintext_key: Final = f"sk-credential-{uuid.uuid4().hex}" + gateway.post( + "/credentials", + {"credential_name": name, "credential_values": {"api_key": plaintext_key}, "credential_info": {}}, + ) + scenario.cleanups.callback(gateway.request, "DELETE", f"/credentials/{name}") + stored: Final = _stored_credential_api_key(name) + assert stored.startswith(V3_PREFIX), f"expected a {V3_PREFIX} ciphertext, stored {stored!r}" + assert plaintext_key not in stored + + +def test_legacy_nacl_model_row_still_decrypts_for_the_upstream_call(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + plaintext_key: Final = f"sk-nacl-{uuid.uuid4().hex}" + _, model_name = _insert_model_row(gateway, scenario, _legacy_nacl_ciphertext(plaintext_key)) + _chat_reaches_upstream_with(gateway, model_name, plaintext_key) + + +def test_v2_gcm_model_row_still_decrypts_for_the_upstream_call(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + plaintext_key: Final = f"sk-v2-{uuid.uuid4().hex}" + _, model_name = _insert_model_row(gateway, scenario, _legacy_v2_ciphertext(plaintext_key)) + _chat_reaches_upstream_with(gateway, model_name, plaintext_key) + + +def test_encryption_check_counts_v3_as_migrated_and_nacl_as_legacy(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + migrated_before: Final = _model_table_count(gateway, "already_v2") + legacy_before: Final = _model_table_count(gateway, "legacy") + _new_model(gateway, scenario, f"sk-upstream-{uuid.uuid4().hex}") + _insert_model_row(gateway, scenario, _legacy_nacl_ciphertext(f"sk-nacl-{uuid.uuid4().hex}")) + _insert_model_row(gateway, scenario, _legacy_v2_ciphertext(f"sk-v2-{uuid.uuid4().hex}")) + migrated_after: Final = _model_table_count(gateway, "already_v2") + assert migrated_after == migrated_before + 3 + 1, ( + "expected every litellm_params value of the new model (model, api_key, api_base) plus the v2 row " + f"counted as migrated, before {migrated_before} after {migrated_after}" + ) + assert _model_table_count(gateway, "legacy") == legacy_before + 1 + + +def test_migrate_encryption_runs_under_default_settings_and_rewrites_legacy_rows(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + plaintext_key: Final = f"sk-nacl-{uuid.uuid4().hex}" + model_id, model_name = _insert_model_row(gateway, scenario, _legacy_nacl_ciphertext(plaintext_key)) + migrated: Final = gateway.request("POST", "/credentials/migrate-encryption", {}) + assert migrated.status_code == 200, migrated.text + rewritten: Final = _stored_api_key(model_id) + assert rewritten.startswith(V3_PREFIX), f"expected a {V3_PREFIX} ciphertext, stored {rewritten!r}" + _chat_reaches_upstream_with(gateway, model_name, plaintext_key) + + +def test_undecryptable_model_row_does_not_stop_other_models_from_serving(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + _insert_model_row(gateway, scenario, base64.b64encode(os.urandom(48)).decode()) + healthy_key: Final = f"sk-upstream-{uuid.uuid4().hex}" + _, healthy_model = _new_model(gateway, scenario, healthy_key) + _chat_reaches_upstream_with(gateway, healthy_model, healthy_key) + assert gateway.request("GET", "/health/liveliness").status_code == 200 + + +def _burst_request(gateway: Gateway, model: str, marker: str) -> tuple[str, int]: + try: + response: Final = gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]} + ) + return marker, response.status_code + except httpx.TransportError: + return marker, 0 + + +def test_proxy_restart_mid_burst_reloads_every_encryption_format_from_the_database( + gateway: Gateway, tmp_path: Path +) -> None: + with gateway.scenario() as scenario: + keys: Final = (f"sk-nacl-{uuid.uuid4().hex}", f"sk-v2-{uuid.uuid4().hex}", f"sk-upstream-{uuid.uuid4().hex}") + _, nacl_model = _insert_model_row(gateway, scenario, _legacy_nacl_ciphertext(keys[0])) + _, v2_model = _insert_model_row(gateway, scenario, _legacy_v2_ciphertext(keys[1])) + _, v3_model = _new_model(gateway, scenario, keys[2]) + models: Final = (nacl_model, v2_model, v3_model) + expected: Final = dict(zip(models, (f"Bearer {key}" for key in keys), strict=True)) + markers: Final = tuple(f"burst-{uuid.uuid4().hex}" for _ in range(24)) + with owned_proxy_process(gateway, tmp_path, {}, workers=2) as first: + for model, key in zip(models, keys, strict=True): + _chat_reaches_upstream_with(first.gateway, model, key) + with ThreadPoolExecutor(max_workers=24) as pool: + futures: Final = tuple( + pool.submit(_burst_request, first.gateway, models[index % 3], marker) + for index, marker in enumerate(markers) + ) + eventually(lambda: sum(future.done() for future in futures), lambda done: done >= 6, seconds=60) + os.killpg(first.process.pid, signal.SIGTERM) + outcomes: Final = tuple(future.result(timeout=120) for future in futures) + served: Final = frozenset(marker for marker, status in outcomes if status == 200) + assert served, outcomes + observed: Final = _drain_upstream(gateway) + for index, marker in enumerate(markers): + seen = _authorizations_for(observed, marker) + assert len(seen) <= 1, (marker, seen) + if marker in served: + assert seen == (expected[models[index % 3]],), (marker, seen) + with owned_proxy_process(gateway, tmp_path, {}, workers=2) as restarted: + for model, key in zip(models, keys, strict=True): + _chat_reaches_upstream_with(restarted.gateway, model, key) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py index bd310339a1d..c181486c8d4 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/outbound_credentials/test_envelope.py @@ -50,7 +50,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import open_refresh_envelope, user_identity, ) -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value, encrypt_value +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with, encrypt_value _NOW = datetime(2026, 7, 9, 12, 0, 0, tzinfo=timezone.utc) _SIGNING_KEY = "unit-test-signing-key-0123456789abcdef0123456789abcdef" @@ -136,7 +136,8 @@ def test_minimal_grant_round_trips_without_none_leakage_into_claims(): claims = _unverified_claims(token) blob = claims["grant"] assert isinstance(blob, str) - plaintext = decrypt_value(value=base64.urlsafe_b64decode(blob), signing_key=_ENCRYPTION_KEY) + plaintext = decrypt_if_encrypted_with(blob, _ENCRYPTION_KEY) + assert plaintext is not None assert set(json.loads(plaintext)) == {"access_token", "token_type"} opened = open_envelope(token, _KEYS, _NOW) assert isinstance(opened, OpenedEnvelope) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index cfcff73b857..9b171c18829 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -472,7 +472,7 @@ async def test_secret_maps_create_update_round_trip(map_algorithm: str, field: s created: Final = await create_mcp_server(prisma, create, touched_by="test") first: Final = table.rows["srv-map"][field] assert isinstance(first, str) and isinstance(json.loads(first), str) - assert json.loads(first).startswith("v2:gcm:") is (map_algorithm == "aes-256-gcm") + assert json.loads(first).startswith("v3:gcm:") is (map_algorithm == "aes-256-gcm") assert "sensitive-secret" not in first and "TEMPLATE" not in first assert getattr(created, field) == original == getattr(create, field) assert decode_secret_map(first, key=field) == original diff --git a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py index 9c07242bd23..442c4706e83 100644 --- a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -1,148 +1,247 @@ """ Tests for the at-rest credential encryption chokepoint. -Covers the AES-256-GCM (``v2:gcm:``) path, the ``encryption_algorithm`` config -gate, and the backward-compatibility guarantees that let legacy XSalsa20-Poly1305 -(nacl) ciphertext and new AES values coexist and decrypt correctly. +Covers the AES-256-GCM default (``v3:gcm:`` written under HKDF-SHA256), the +``encryption_algorithm`` legacy opt-in and its FIPS refusal, and the dual-read +guarantees that keep ``v2:gcm:`` (raw SHA-256 AES) and unprefixed +XSalsa20-Poly1305 (nacl) ciphertext decrypting after the default flip. """ import base64 +import hashlib +import os +import sys import pytest +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM from litellm.proxy import proxy_server from litellm.proxy.common_utils.encrypt_decrypt_utils import ( _V2_GCM_PREFIX, + _V3_GCM_PREFIX, + LegacyEncryptionUnavailableError, + _get_encryption_algorithm, decrypt_if_encrypted_with, decrypt_value_helper, encrypt_value, encrypt_value_helper, + is_versioned_gcm, ) +from litellm.proxy.common_utils.fips import FipsModeError + +_SALT_KEY = "sk-salt-aes-1234" -def _use_aes(monkeypatch): - """Flip the write-time algorithm to AES-256-GCM for the duration of a test.""" - monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "aes-256-gcm"}) +def _use_legacy(monkeypatch): + """Opt the write-time algorithm back into XSalsa20-Poly1305 for the duration of a test.""" + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) + + +def _sha256_key(signing_key: str = _SALT_KEY) -> bytes: + return hashlib.sha256(signing_key.encode()).digest() + + +def _hkdf_key(signing_key: str = _SALT_KEY) -> bytes: + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + return HKDF(algorithm=hashes.SHA256(), length=32, salt=None, info=b"litellm-at-rest-v3").derive( + signing_key.encode() + ) + + +def _v2_ciphertext(plaintext: str, key: bytes) -> str: + """Build a ``v2:gcm:`` value the way the previous release wrote it: AES-GCM under raw SHA-256.""" + nonce = os.urandom(12) + return ( + _V2_GCM_PREFIX + base64.urlsafe_b64encode(nonce + AESGCM(key).encrypt(nonce, plaintext.encode(), None)).decode() + ) + + +def _legacy_nacl_ciphertext(plaintext: str, key: bytes) -> str: + """Build an unprefixed value the way pre-AES releases wrote it: XSalsa20-Poly1305 under raw SHA-256.""" + import nacl.secret + + return base64.urlsafe_b64encode(bytes(nacl.secret.SecretBox(key).encrypt(plaintext.encode()))).decode() @pytest.fixture(autouse=True) def _salt_key(monkeypatch): - # Dominant convention in the test_litellm/ tree: set the key via env. - monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-aes-1234") - # Ensure the legacy default is in force unless a test opts into AES. + monkeypatch.setenv("LITELLM_SALT_KEY", _SALT_KEY) + monkeypatch.delenv("LITELLM_FIPS_MODE", raising=False) monkeypatch.setattr(proxy_server, "general_settings", {}) yield -def test_aes_gcm_round_trip(monkeypatch): - """A value written under AES-256-GCM is tagged v2:gcm: and decrypts back.""" - _use_aes(monkeypatch) - +def test_default_write_is_v3_gcm_and_round_trips(): ct = encrypt_value_helper("super-secret") - assert ct.startswith(_V2_GCM_PREFIX) + assert ct.startswith(_V3_GCM_PREFIX), ct assert decrypt_value_helper(ct, key="t") == "super-secret" -def test_default_is_legacy_algorithm(monkeypatch): - """With no config, writes stay on the legacy algorithm (no v2: marker).""" - ct = encrypt_value_helper("legacy-secret") +def test_explicit_aes_setting_writes_v3_gcm(monkeypatch): + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "AES-256-GCM"}) - assert not ct.startswith(_V2_GCM_PREFIX) - assert decrypt_value_helper(ct, key="t") == "legacy-secret" + assert encrypt_value_helper("secret").startswith(_V3_GCM_PREFIX) -def test_legacy_nacl_value_still_decrypts_after_flag_flip(monkeypatch): - """A value written under the old algorithm decrypts unchanged once AES is on. - - This is the mixed-format readback guarantee: decrypt is format-detecting, so - flipping the flag forward never strands previously-written data. - """ - legacy = encrypt_value_helper("legacy-secret") # default = xsalsa20 - assert not legacy.startswith(_V2_GCM_PREFIX) - - _use_aes(monkeypatch) - # New writes are now AES, but the old value must still come back. - assert decrypt_value_helper(legacy, key="t") == "legacy-secret" - assert encrypt_value_helper("fresh").startswith(_V2_GCM_PREFIX) - - -def test_v2_prefix_is_idempotent_marker(monkeypatch): - """The migration's skip-check: an already-v2 value is recognized by its prefix. - - Re-encrypting an AES value yields a fresh (different nonce) AES value, but the - prefix is what lets a migration skip already-migrated rows without decrypting. - """ - _use_aes(monkeypatch) +def test_unknown_algorithm_falls_back_to_the_aes_default(monkeypatch): + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "rot13"}) ct = encrypt_value_helper("secret") - assert ct.startswith(_V2_GCM_PREFIX) + assert ct.startswith(_V3_GCM_PREFIX), ct + assert decrypt_value_helper(ct, key="t") == "secret" + + +def test_v3_value_is_aes_gcm_under_hkdf_sha256_not_raw_sha256(): + ct = encrypt_value_helper("hkdf-secret") + raw = base64.urlsafe_b64decode(ct[len(_V3_GCM_PREFIX) :]) + nonce, blob = raw[:12], raw[12:] + + assert AESGCM(_hkdf_key()).decrypt(nonce, blob, None) == b"hkdf-secret" + with pytest.raises(InvalidTag): + AESGCM(_sha256_key()).decrypt(nonce, blob, None) + + +def test_each_v3_write_uses_a_fresh_nonce(): + first, second = encrypt_value_helper("same-secret"), encrypt_value_helper("same-secret") + + assert first != second + assert ( + base64.urlsafe_b64decode(first[len(_V3_GCM_PREFIX) :])[:12] + != base64.urlsafe_b64decode(second[len(_V3_GCM_PREFIX) :])[:12] + ) + + +def test_v2_value_written_under_raw_sha256_still_decrypts(): + legacy_v2 = _v2_ciphertext("v2-secret", _sha256_key()) + + assert decrypt_value_helper(legacy_v2, key="t") == "v2-secret" + assert decrypt_if_encrypted_with(legacy_v2, _SALT_KEY) == "v2-secret" + + +def test_v2_value_is_not_read_with_the_v3_derivation(): + """A v2 body relabelled as v3 must fail: the prefix, not the caller, picks the derivation.""" + v2_body = _v2_ciphertext("v2-secret", _sha256_key())[len(_V2_GCM_PREFIX) :] + + assert decrypt_if_encrypted_with(_V3_GCM_PREFIX + v2_body, _SALT_KEY) is None + assert decrypt_value_helper(_V3_GCM_PREFIX + v2_body, key="t", exception_type="debug") is None + + +def test_legacy_nacl_value_still_decrypts_under_the_aes_default(): + legacy = _legacy_nacl_ciphertext("legacy-secret", _sha256_key()) + assert not is_versioned_gcm(legacy) + + assert decrypt_value_helper(legacy, key="t") == "legacy-secret" + assert decrypt_if_encrypted_with(legacy, _SALT_KEY) == "legacy-secret" + assert encrypt_value_helper("fresh").startswith(_V3_GCM_PREFIX) + + +def test_legacy_opt_in_writes_unprefixed_nacl_that_still_reads_back(monkeypatch): + _use_legacy(monkeypatch) + + ct = encrypt_value_helper("legacy-secret") + + assert not is_versioned_gcm(ct), ct + assert decrypt_value_helper(ct, key="t") == "legacy-secret" + assert _get_encryption_algorithm() == "xsalsa20-poly1305" + + +def test_legacy_opt_in_is_refused_under_fips_mode(monkeypatch): + _use_legacy(monkeypatch) + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + with pytest.raises(FipsModeError, match="xsalsa20-poly1305") as refused: + encrypt_value_helper("secret") + assert "aes-256-gcm" in str(refused.value) + + +def test_fips_mode_with_the_default_setting_writes_v3_gcm(monkeypatch): + monkeypatch.setenv("LITELLM_FIPS_MODE", "true") + + ct = encrypt_value_helper("fips-secret") + assert ct.startswith(_V3_GCM_PREFIX), ct + assert decrypt_value_helper(ct, key="t") == "fips-secret" + + +def test_versioned_gcm_values_never_import_nacl(monkeypatch): + legacy_v2 = _v2_ciphertext("v2-secret", _sha256_key()) + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + assert decrypt_value_helper(encrypt_value_helper("v3-secret"), key="t") == "v3-secret" + assert decrypt_value_helper(legacy_v2, key="t") == "v2-secret" + + +def test_legacy_ciphertext_without_pynacl_names_the_reencrypt_path(monkeypatch): + legacy = _legacy_nacl_ciphertext("legacy-secret", _sha256_key()) + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + with pytest.raises(LegacyEncryptionUnavailableError) as missing: + decrypt_value_helper(legacy, key="t", exception_type="debug", return_original_value=True) + message = str(missing.value) + assert "PyNaCl is not installed" in message + assert "legacy-encryption" in message + assert "/credentials/migrate-encryption" in message + + with pytest.raises(LegacyEncryptionUnavailableError): + decrypt_if_encrypted_with(legacy, _SALT_KEY) + + +def test_legacy_opt_in_without_pynacl_fails_the_write_not_silently(monkeypatch): + _use_legacy(monkeypatch) + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + with pytest.raises(LegacyEncryptionUnavailableError, match="encrypt"): + encrypt_value_helper("secret") + + +def test_v3_prefix_is_the_idempotent_migration_marker(): + ct = encrypt_value_helper("secret") + assert is_versioned_gcm(ct) - # Round-tripping does not change the plaintext, and the marker is stable. again = encrypt_value_helper(decrypt_value_helper(ct, key="t")) - assert again.startswith(_V2_GCM_PREFIX) + assert again.startswith(_V3_GCM_PREFIX) assert decrypt_value_helper(again, key="t") == "secret" -def test_aes_decrypt_failure_returns_none_not_raise(monkeypatch): - """Decrypt contract preserved: a garbled v2 value returns None, never raises.""" - _use_aes(monkeypatch) - - garbled = _V2_GCM_PREFIX + "not-valid-base64-or-ciphertext!!!" - # exception_type="debug" exercises the swallow path; must not raise. +@pytest.mark.parametrize("prefix", [_V3_GCM_PREFIX, _V2_GCM_PREFIX]) +def test_aes_decrypt_failure_returns_none_not_raise(prefix: str): + garbled = prefix + "not-valid-base64-or-ciphertext!!!" assert decrypt_value_helper(garbled, key="t", exception_type="debug") is None -def test_aes_decrypt_failure_returns_original_when_requested(monkeypatch): - """With return_original_value=True a bad v2 value comes back as-is, not None.""" - _use_aes(monkeypatch) - - garbled = _V2_GCM_PREFIX + "###" +def test_aes_decrypt_failure_returns_original_when_requested(): + garbled = _V3_GCM_PREFIX + "###" assert decrypt_value_helper(garbled, key="t", exception_type="debug", return_original_value=True) == garbled -def test_empty_string_round_trips_under_aes(monkeypatch): - """Empty string is preserved through the AES path (parity with legacy).""" - _use_aes(monkeypatch) - +def test_empty_string_round_trips_under_aes(): ct = encrypt_value_helper("") - assert ct.startswith(_V2_GCM_PREFIX) + assert ct.startswith(_V3_GCM_PREFIX) assert decrypt_value_helper(ct, key="t") == "" -def test_callback_prefix_composes_with_v2(monkeypatch): - """litellm_enc:: + v2:gcm:... round-trips through the callback read path. - - Callback vars are stored as ``litellm_enc::``; the read path - strips ``litellm_enc::`` then calls the helper, so the value handed to the - helper is ``v2:gcm:...``. Ordering must work end to end. - """ +def test_callback_prefix_composes_with_v3(): + """litellm_enc:: + v3:gcm:... round-trips through the callback read path.""" from litellm.proxy.common_utils.callback_utils import ( _CALLBACK_VAR_ENCRYPTED_PREFIX, _decrypt_or_passthrough, _encrypt_if_plaintext, ) - _use_aes(monkeypatch) - - # "gcs_path_service_account" is a known-sensitive callback key. stored = _encrypt_if_plaintext("gcs_path_service_account", "my-sa-secret") assert stored.startswith(_CALLBACK_VAR_ENCRYPTED_PREFIX) inner = stored[len(_CALLBACK_VAR_ENCRYPTED_PREFIX) :] - assert inner.startswith(_V2_GCM_PREFIX) + assert inner.startswith(_V3_GCM_PREFIX) assert _decrypt_or_passthrough("gcs_path_service_account", stored) == "my-sa-secret" -def test_unknown_algorithm_falls_back_to_legacy(monkeypatch): - """An unrecognized encryption_algorithm value does not produce v2 writes.""" - monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "rot13"}) - - ct = encrypt_value_helper("secret") - assert not ct.startswith(_V2_GCM_PREFIX) - assert decrypt_value_helper(ct, key="t") == "secret" - - def test_decrypt_failure_debug_log_omits_raw_value(monkeypatch): """Regression for LIT-4152: the decrypt-failure debug breadcrumb must not embed the raw value. @@ -191,10 +290,10 @@ def test_decrypt_failure_debug_log_omits_raw_value(monkeypatch): assert result == secret -@pytest.mark.parametrize("use_aes", [False, True]) -def test_explicit_key_decrypt_reads_only_values_written_under_that_key(monkeypatch, use_aes: bool): - if use_aes: - _use_aes(monkeypatch) +@pytest.mark.parametrize("use_legacy", [False, True]) +def test_explicit_key_decrypt_reads_only_values_written_under_that_key(monkeypatch, use_legacy: bool): + if use_legacy: + _use_legacy(monkeypatch) written_with_previous_key = encrypt_value_helper("stored-secret", new_encryption_key="sk-1234") assert decrypt_if_encrypted_with(written_with_previous_key, "sk-1234") == "stored-secret" @@ -209,6 +308,7 @@ def test_explicit_key_decrypt_reads_only_values_written_under_that_key(monkeypat "gpt-5.4-mini", "https://example.invalid/v1", "v2:gcm:", + "v3:gcm:", "aGVsbG8=", "*", "-", @@ -224,10 +324,10 @@ def test_explicit_key_decrypt_rejects_values_that_are_not_ciphertexts(not_a_ciph assert decrypt_if_encrypted_with(not_a_ciphertext, "sk-1234") is None -@pytest.mark.parametrize("use_aes", [False, True]) -def test_explicit_key_decrypt_tells_an_encrypted_empty_string_from_no_ciphertext(monkeypatch, use_aes: bool): - if use_aes: - _use_aes(monkeypatch) +@pytest.mark.parametrize("use_legacy", [False, True]) +def test_explicit_key_decrypt_tells_an_encrypted_empty_string_from_no_ciphertext(monkeypatch, use_legacy: bool): + if use_legacy: + _use_legacy(monkeypatch) assert decrypt_if_encrypted_with(encrypt_value_helper("", new_encryption_key="sk-1234"), "sk-1234") == "" diff --git a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py index 0ecc4f8d7cb..07befdb19a5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py +++ b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py @@ -15,7 +15,7 @@ import pytest from litellm.proxy import proxy_server from litellm.proxy.common_utils.encrypt_decrypt_utils import ( - _V2_GCM_PREFIX, + _V3_GCM_PREFIX, encrypt_value_helper, ) from litellm.proxy.management_endpoints import credential_migration as cm @@ -29,8 +29,8 @@ def salt_key(monkeypatch): def _legacy_ct(value: str, monkeypatch) -> str: - """Produce a legacy (nacl) ciphertext with the AES gate off.""" - monkeypatch.setattr(proxy_server, "general_settings", {}) + """Produce a legacy (nacl) ciphertext through the explicit xsalsa20-poly1305 opt-in.""" + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) return encrypt_value_helper(value) @@ -79,7 +79,7 @@ def test_reencrypt_value_legacy_to_v2(salt_key, monkeypatch): out = cm.reencrypt_value(legacy) assert out != legacy - assert out.startswith(_V2_GCM_PREFIX) + assert out.startswith(_V3_GCM_PREFIX) def test_reencrypt_value_is_idempotent(salt_key, monkeypatch): @@ -110,7 +110,7 @@ def test_reencrypt_selective_dict(salt_key, monkeypatch): data = {"api_key": legacy_key, "base_url": "https://x", "integration_token": None} out = cm.reencrypt_selective_dict(data, ["api_key", "integration_token"]) - assert out["api_key"].startswith(_V2_GCM_PREFIX) + assert out["api_key"].startswith(_V3_GCM_PREFIX) assert out["base_url"] == "https://x" # untouched non-sensitive assert out["integration_token"] is None # null skipped @@ -120,7 +120,7 @@ def test_reencrypt_selective_dict(salt_key, monkeypatch): @pytest.mark.asyncio async def test_migrate_requires_aes_gate(salt_key, monkeypatch): - monkeypatch.setattr(proxy_server, "general_settings", {}) # gate off + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) # legacy opt-in with pytest.raises(RuntimeError, match="encryption_algorithm"): await cm.migrate_encryption( prisma_client=MagicMock(), user_api_key_dict=MagicMock() @@ -161,7 +161,7 @@ async def test_vantage_walker_migrates_legacy_field(salt_key, monkeypatch): written = json.loads( client.db.litellm_config.update.call_args.kwargs["data"]["param_value"] ) - assert written["api_key"].startswith(_V2_GCM_PREFIX) + assert written["api_key"].startswith(_V3_GCM_PREFIX) assert written["base_url"] == "https://api.vantage.sh" # non-sensitive untouched @@ -305,8 +305,8 @@ async def test_callback_vars_walker_migrates_team_metadata(salt_key, monkeypatch """A team row with a legacy-encrypted callback var is rewritten to v2.""" from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars - # Legacy-encrypt a callback var via the real callback path (gate off). - monkeypatch.setattr(proxy_server, "general_settings", {}) + # Legacy-encrypt a callback var via the real callback path (legacy opt-in). + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) legacy_meta = encrypt_callback_vars( {"logging": [{"callback_vars": {"gcs_path_service_account": "sa-secret"}}]} ) @@ -326,7 +326,7 @@ async def test_callback_vars_walker_migrates_team_metadata(salt_key, monkeypatch client.db.litellm_teamtable.update.call_args.kwargs["data"]["metadata"] ) inner = written["logging"][0]["callback_vars"]["gcs_path_service_account"] - assert "v2:gcm:" in inner + assert "v3:gcm:" in inner @pytest.mark.asyncio @@ -334,7 +334,7 @@ async def test_callback_vars_walker_dry_run_reports_legacy(salt_key, monkeypatch """In --check (dry-run) mode, a legacy callback var counts as residual legacy.""" from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars - monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) legacy_meta = encrypt_callback_vars( {"logging": [{"callback_vars": {"gcs_path_service_account": "sa-secret"}}]} ) @@ -366,7 +366,7 @@ async def test_callback_vars_walker_migrates_callback_settings_shape( """ from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars - monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) legacy_meta = encrypt_callback_vars( { "callback_settings": { @@ -391,7 +391,7 @@ async def test_callback_vars_walker_migrates_callback_settings_shape( client.db.litellm_teamtable.update.call_args.kwargs["data"]["metadata"] ) inner = written["callback_settings"]["callback_vars"]["gcs_path_service_account"] - assert "v2:gcm:" in inner + assert "v3:gcm:" in inner @pytest.mark.asyncio @@ -401,13 +401,13 @@ async def test_check_reports_callback_var_legacy_with_gate_off(salt_key, monkeyp Detection is decrypt-based, not a re-encrypt delta, so it does not depend on the write gate. A heuristic that re-encrypts and counts new v2 values would - read zero here (gate off -> no v2 produced) and emit a false-clean + read zero here (legacy opt-in -> no versioned AES produced) and emit a false-clean attestation -- exactly the compliance trap this guards against. """ from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars - # Legacy-encrypt a callback var, and leave the gate OFF for the check itself. - monkeypatch.setattr(proxy_server, "general_settings", {}) + # Legacy-encrypt a callback var, and keep the legacy opt-in for the check itself. + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) legacy_meta = encrypt_callback_vars( {"logging": [{"callback_vars": {"gcs_path_service_account": "sa-secret"}}]} ) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2938e65cfde..bda8fac23ce 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -3646,9 +3646,9 @@ export interface paths { put?: never; /** * Migrate Encryption Endpoint - * @description Re-encrypt all at-rest credentials into the AES-256-GCM (``v2:gcm:``) format. + * @description Re-encrypt all at-rest credentials into the versioned AES-256-GCM (``v3:gcm:``) format. * - * Admin only. Requires ``general_settings.encryption_algorithm: aes-256-gcm``. + * Admin only. Requires the proxy to write ``aes-256-gcm`` (the default). * Idempotent and resumable — re-running skips already-migrated values. Pass * ``dry_run=true`` for a non-mutating scan (equivalent to ``--check``). */ diff --git a/uv.lock b/uv.lock index 85e2b6d4e52..cd0e564ccca 100644 --- a/uv.lock +++ b/uv.lock @@ -4555,6 +4555,9 @@ google = [ grpc = [ { name = "grpcio" }, ] +legacy-encryption = [ + { name = "pynacl" }, +] mcp = [ { name = "httpx2" }, { name = "mcp" }, @@ -4585,7 +4588,6 @@ proxy = [ { name = "polars" }, { name = "pydantic" }, { name = "pyjwt" }, - { name = "pynacl" }, { name = "pyroscope-io", marker = "sys_platform != 'win32'" }, { name = "python-multipart" }, { name = "pyyaml" }, @@ -4823,7 +4825,7 @@ requires-dist = [ { name = "pydantic", marker = "extra == 'proxy'", specifier = ">=2.12.0,<3" }, { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" }, - { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" }, + { name = "pynacl", marker = "extra == 'legacy-encryption'", specifier = ">=1.6.2,<2.0" }, { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.16.1,<7.0" }, { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" }, { name = "python-dotenv", specifier = ">=1.0.0,<2.0" }, @@ -4853,7 +4855,7 @@ requires-dist = [ { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, ] -provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] +provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "legacy-encryption", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-vertex-chirp", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"] [package.metadata.requires-dev] benchmarks = [ From e00c199779c8ee0b152583d24f429fc44d225a59 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 08:33:50 +0000 Subject: [PATCH 06/17] build(deps): keep pynacl in the proxy-dev group so legacy ciphertext fixtures run on every CI shard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- pyproject.toml | 1 + .../proxy/_experimental/mcp_server/test_db_credentials.py | 2 +- uv.lock | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d51aefaa607..a6e0873f598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -256,6 +256,7 @@ proxy-dev = [ "opentelemetry-instrumentation-fastapi==0.49b0", "azure-identity==1.25.2", "a2a-sdk==1.1.0", + "pynacl>=1.6.2,<2.0", ] ci = [ # These are lazily imported at call sites; keep them out of core deps to diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py index 9b171c18829..4b6062b52da 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_db_credentials.py @@ -4,7 +4,7 @@ Tests for the encrypted-at-rest persistence of MCP user credentials. The ``LiteLLM_MCPUserCredentials.credential_b64`` column previously stored both BYOK API keys and OAuth2 access tokens as plain ``urlsafe_b64encode`` of the raw value, leaving credentials readable from any DB read. The fix -runs every write through ``encrypt_value_helper`` (nacl SecretBox) and +runs every write through ``encrypt_value_helper`` (AES-256-GCM by default) and keeps a plain-base64 fallback on read so existing rows continue to work. """ diff --git a/uv.lock b/uv.lock index cd0e564ccca..219b0a56981 100644 --- a/uv.lock +++ b/uv.lock @@ -4744,6 +4744,7 @@ proxy-dev = [ { name = "opentelemetry-sdk" }, { name = "prisma" }, { name = "prometheus-client" }, + { name = "pynacl" }, ] [package.metadata] @@ -4957,6 +4958,7 @@ proxy-dev = [ { name = "opentelemetry-sdk", specifier = "==1.28.0" }, { name = "prisma", specifier = "==0.11.0" }, { name = "prometheus-client", specifier = "==0.20.0" }, + { name = "pynacl", specifier = ">=1.6.2,<2.0" }, ] [[package]] From 4d15b98f49a2957a653e948e6d234f2b1eab6d3f Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 10:24:05 +0000 Subject: [PATCH 07/17] fix(proxy): log missing PyNaCl per value instead of aborting the whole decrypt batch Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../common_utils/encrypt_decrypt_utils.py | 14 +++++++----- .../test_encrypt_decrypt_utils.py | 22 ++++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 9f91644abb2..ed24bc1fbb3 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -165,16 +165,17 @@ def _decrypt_with_signing_key(value: str, signing_key: str) -> str: def decrypt_if_encrypted_with(value: str, signing_key: str) -> str | None: """None unless value is a ciphertext under signing_key. - A legacy ciphertext met without PyNaCl installed is not silently "not a ciphertext": it raises - :class:`LegacyEncryptionUnavailableError` naming the re-encrypt path. + A legacy ciphertext met without PyNaCl installed is logged with the re-encrypt path and read as None, + never handed back as if it were the plaintext. """ try: # base64 decoding skips characters outside its alphabet, so "" and "*" decode to no bytes, # which decrypt_value reads as an empty plaintext under any key. decodes_to_nothing: Final = not is_versioned_gcm(value) and not _legacy_ciphertext_bytes(value) return None if decodes_to_nothing else _decrypt_with_signing_key(value=value, signing_key=signing_key) - except LegacyEncryptionUnavailableError: - raise + except LegacyEncryptionUnavailableError as error: + verbose_proxy_logger.error("%s", error) + return None except Exception: # noqa: BLE001 # base64, nacl and AES-GCM each raise their own "not a ciphertext" type return None @@ -193,8 +194,9 @@ def decrypt_value_helper( # if it's not str - do not decrypt it, return the value return value - except LegacyEncryptionUnavailableError: - raise + except LegacyEncryptionUnavailableError as error: + verbose_proxy_logger.error("Cannot decrypt value for key: %s. %s", key, error) + return None except Exception as e: error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {e}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" if exception_type == "debug": diff --git a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py index 442c4706e83..571d8b6a8ca 100644 --- a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -9,6 +9,7 @@ XSalsa20-Poly1305 (nacl) ciphertext decrypting after the default flip. import base64 import hashlib +import logging import os import sys @@ -175,20 +176,21 @@ def test_versioned_gcm_values_never_import_nacl(monkeypatch): assert decrypt_value_helper(legacy_v2, key="t") == "v2-secret" -def test_legacy_ciphertext_without_pynacl_names_the_reencrypt_path(monkeypatch): +def test_legacy_ciphertext_without_pynacl_logs_the_reencrypt_path_and_reads_as_none(monkeypatch, caplog): legacy = _legacy_nacl_ciphertext("legacy-secret", _sha256_key()) monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) - with pytest.raises(LegacyEncryptionUnavailableError) as missing: - decrypt_value_helper(legacy, key="t", exception_type="debug", return_original_value=True) - message = str(missing.value) - assert "PyNaCl is not installed" in message - assert "legacy-encryption" in message - assert "/credentials/migrate-encryption" in message - - with pytest.raises(LegacyEncryptionUnavailableError): - decrypt_if_encrypted_with(legacy, _SALT_KEY) + with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): + assert decrypt_value_helper(legacy, key="t", exception_type="debug", return_original_value=True) is None + assert decrypt_if_encrypted_with(legacy, _SALT_KEY) is None + assert len(caplog.records) == 2, caplog.text + for record in caplog.records: + message = record.getMessage() + assert "PyNaCl is not installed" in message + assert "legacy-encryption" in message + assert "/credentials/migrate-encryption" in message + assert "legacy-secret" not in caplog.text def test_legacy_opt_in_without_pynacl_fails_the_write_not_silently(monkeypatch): From 016e703662ce44c16a77895a4783e01e1a1816e3 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 10:46:19 +0000 Subject: [PATCH 08/17] fix(proxy): keep stored legacy ciphertext when PyNaCl is missing and refuse encryption migration without it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/common_utils/encrypt_decrypt_utils.py | 11 ++++++++++- .../management_endpoints/credential_migration.py | 13 +++++++++++++ .../common_utils/test_encrypt_decrypt_utils.py | 10 +++++++--- .../test_credential_migration.py | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index ed24bc1fbb3..59454a9ecae 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -196,7 +196,7 @@ def decrypt_value_helper( return value except LegacyEncryptionUnavailableError as error: verbose_proxy_logger.error("Cannot decrypt value for key: %s. %s", key, error) - return None + return value if return_original_value else None except Exception as e: error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {e}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" if exception_type == "debug": @@ -212,6 +212,15 @@ def decrypt_value_helper( return None +def legacy_encryption_available() -> bool: + """True when PyNaCl is importable, so legacy xsalsa20-poly1305 ciphertext can be read.""" + try: + import nacl.secret # noqa: F401 # probe only + except ImportError: + return False + return True + + def _legacy_secret_box(signing_key: str, purpose: str) -> "SecretBox": try: import nacl.secret diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index 181a0e2b016..6996c379c7e 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -42,6 +42,7 @@ if TYPE_CHECKING: from litellm.proxy.common_utils.encrypt_decrypt_utils import ( _ALGO_AES_GCM, _ENCRYPTION_ALGORITHM_SETTING, + _LEGACY_ENCRYPTION_HELP, SecretMapDecodeError, _get_encryption_algorithm, _get_salt_key, @@ -49,6 +50,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, is_versioned_gcm, + legacy_encryption_available, ) ValueClass = Literal["migrated", "legacy", "plaintext", "undecryptable", "not-a-string"] @@ -204,6 +206,15 @@ def _assert_aes_gate_enabled() -> None: ) +def _assert_legacy_reader_available() -> None: + """Fail fast when PyNaCl is missing: without it every unprefixed value is unreadable and would be + miscounted as plaintext, so the scan cannot attest anything and the migration cannot rewrite legacy rows.""" + if not legacy_encryption_available(): + raise RuntimeError( + f"Encryption scan and migration need PyNaCl to read legacy values. {_LEGACY_ENCRYPTION_HELP}" + ) + + # --------------------------------------------------------------------------- # Walkers for the locations with no pre-existing rotation path. # Each walker delegates the structural transform to the existing, tested helper @@ -633,6 +644,7 @@ async def migrate_encryption( net-new walkers run in dry-run mode. """ _assert_aes_gate_enabled() + _assert_legacy_reader_available() report: Final = MigrationReport() @@ -665,6 +677,7 @@ async def check_encryption(prisma_client: object) -> MigrationReport: config rows, SSO config). Reports how many values are still ``legacy``; ``residual_legacy == 0`` across this full scan is the compliance attestation. """ + _assert_legacy_reader_available() report: Final = MigrationReport() # Rotation-covered tables (read-only classification). diff --git a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py index 571d8b6a8ca..f12f000ee1d 100644 --- a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -28,6 +28,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value, encrypt_value_helper, is_versioned_gcm, + legacy_encryption_available, ) from litellm.proxy.common_utils.fips import FipsModeError @@ -176,15 +177,18 @@ def test_versioned_gcm_values_never_import_nacl(monkeypatch): assert decrypt_value_helper(legacy_v2, key="t") == "v2-secret" -def test_legacy_ciphertext_without_pynacl_logs_the_reencrypt_path_and_reads_as_none(monkeypatch, caplog): +def test_legacy_ciphertext_without_pynacl_logs_the_reencrypt_path_and_keeps_the_stored_value(monkeypatch, caplog): legacy = _legacy_nacl_ciphertext("legacy-secret", _sha256_key()) + assert legacy_encryption_available() monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) + assert not legacy_encryption_available() with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): - assert decrypt_value_helper(legacy, key="t", exception_type="debug", return_original_value=True) is None + assert decrypt_value_helper(legacy, key="t", exception_type="debug", return_original_value=True) == legacy + assert decrypt_value_helper(legacy, key="t", exception_type="debug") is None assert decrypt_if_encrypted_with(legacy, _SALT_KEY) is None - assert len(caplog.records) == 2, caplog.text + assert len(caplog.records) == 3, caplog.text for record in caplog.records: message = record.getMessage() assert "PyNaCl is not installed" in message diff --git a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py index 07befdb19a5..4c4ee2e95de 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py +++ b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py @@ -7,6 +7,7 @@ proof-of-fix (real proxy + DB) is performed separately on the repro server. """ import json +import sys from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -127,6 +128,21 @@ async def test_migrate_requires_aes_gate(salt_key, monkeypatch): ) +@pytest.mark.asyncio +async def test_migrate_and_check_refuse_without_pynacl_instead_of_miscounting_legacy_rows(salt_key, monkeypatch): + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + client: Final = MagicMock() + _empty_covered_tables(client) + + with pytest.raises(RuntimeError, match="legacy-encryption"): + await cm.migrate_encryption(prisma_client=client, user_api_key_dict=MagicMock()) + with pytest.raises(RuntimeError, match="legacy-encryption"): + await cm.check_encryption(prisma_client=client) + for _, db_attr, _, _ in cm._COVERED_TABLE_SPECS: + getattr(client.db, db_attr).find_many.assert_not_awaited() + + # --------------------------- config-row walker --------------------------- From 773d7efd148198d4c3b6447665db8a8d9673f3c1 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 11:08:16 +0000 Subject: [PATCH 09/17] fix(proxy): never return unreadable legacy ciphertext as plaintext and refuse rewrite passes without PyNaCl Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../common_utils/encrypt_decrypt_utils.py | 11 ++++++- .../credential_migration.py | 16 ++-------- .../key_management_endpoints.py | 9 ++++++ litellm/proxy/proxy_server.py | 2 ++ .../test_encrypt_decrypt_utils.py | 14 +++++++-- .../test_key_management_endpoints.py | 30 +++++++++++++++++++ .../proxy/proxy_server/test_proxy_config.py | 20 +++++++++++-- 7 files changed, 83 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 59454a9ecae..444a8590ef4 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -196,7 +196,7 @@ def decrypt_value_helper( return value except LegacyEncryptionUnavailableError as error: verbose_proxy_logger.error("Cannot decrypt value for key: %s. %s", key, error) - return value if return_original_value else None + return None except Exception as e: error_message = f"Error decrypting value for key: {key}, Did your master_key/salt key change recently? \nError: {e}\nSet permanent salt key - https://docs.litellm.ai/docs/proxy/prod#5-set-litellm-salt-key" if exception_type == "debug": @@ -221,6 +221,15 @@ def legacy_encryption_available() -> bool: return True +def require_legacy_reader(purpose: str) -> None: + """Refuse a decrypt-then-rewrite pass when PyNaCl is missing: every unprefixed value would read as + unreadable and be dropped, double wrapped or miscounted as plaintext, so the pass cannot be trusted.""" + if not legacy_encryption_available(): + raise LegacyEncryptionUnavailableError( + f"Cannot {purpose}: PyNaCl is needed to read legacy {_ALGO_XSALSA20} values. {_LEGACY_ENCRYPTION_HELP}" + ) + + def _legacy_secret_box(signing_key: str, purpose: str) -> "SecretBox": try: import nacl.secret diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index 6996c379c7e..0cbdb4f0a78 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -42,7 +42,6 @@ if TYPE_CHECKING: from litellm.proxy.common_utils.encrypt_decrypt_utils import ( _ALGO_AES_GCM, _ENCRYPTION_ALGORITHM_SETTING, - _LEGACY_ENCRYPTION_HELP, SecretMapDecodeError, _get_encryption_algorithm, _get_salt_key, @@ -50,7 +49,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, is_versioned_gcm, - legacy_encryption_available, + require_legacy_reader, ) ValueClass = Literal["migrated", "legacy", "plaintext", "undecryptable", "not-a-string"] @@ -206,15 +205,6 @@ def _assert_aes_gate_enabled() -> None: ) -def _assert_legacy_reader_available() -> None: - """Fail fast when PyNaCl is missing: without it every unprefixed value is unreadable and would be - miscounted as plaintext, so the scan cannot attest anything and the migration cannot rewrite legacy rows.""" - if not legacy_encryption_available(): - raise RuntimeError( - f"Encryption scan and migration need PyNaCl to read legacy values. {_LEGACY_ENCRYPTION_HELP}" - ) - - # --------------------------------------------------------------------------- # Walkers for the locations with no pre-existing rotation path. # Each walker delegates the structural transform to the existing, tested helper @@ -644,7 +634,7 @@ async def migrate_encryption( net-new walkers run in dry-run mode. """ _assert_aes_gate_enabled() - _assert_legacy_reader_available() + require_legacy_reader("migrate stored encryption") report: Final = MigrationReport() @@ -677,7 +667,7 @@ async def check_encryption(prisma_client: object) -> MigrationReport: config rows, SSO config). Reports how many values are still ``legacy``; ``residual_legacy == 0`` across this full scan is the compliance attestation. """ - _assert_legacy_reader_available() + require_legacy_reader("scan stored encryption") report: Final = MigrationReport() # Rotation-covered tables (read-only classification). diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index e4d3b7dbffc..d2488c08e6a 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -79,6 +79,10 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( coordination_redis_cache, publish_config_change, ) +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + LegacyEncryptionUnavailableError, + require_legacy_reader, +) from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -5175,6 +5179,11 @@ async def _rotate_master_key( from litellm.proxy.proxy_server import proxy_config + try: + require_legacy_reader("rotate the master key") + except LegacyEncryptionUnavailableError as error: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail={"error": str(error)}) from error + try: models: list | None = cast( # cast-ok: find_many returns a real list, which TableActions widens to Sequence "list[object]", await _prisma_table(ModelRepository(prisma_client)).find_many() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a563b901fc4..e2524148597 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -403,6 +403,7 @@ from litellm.proxy.common_utils.discoverable_model_filter import discoverable_ro from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, + require_legacy_reader, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.fips import ( @@ -7325,6 +7326,7 @@ class ProxyConfig: _decrypt_and_set_db_env_variables): this is a write path, and loading values into os.environ is the read path's responsibility. """ + require_legacy_reader("re-encrypt environment variables for the config save") decrypted_env_vars: Final = self._decrypt_db_variables(environment_variables) return self._encrypt_env_variables( environment_variables=decrypted_env_vars, diff --git a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py index f12f000ee1d..6c0118576ea 100644 --- a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -29,6 +29,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, is_versioned_gcm, legacy_encryption_available, + require_legacy_reader, ) from litellm.proxy.common_utils.fips import FipsModeError @@ -177,7 +178,7 @@ def test_versioned_gcm_values_never_import_nacl(monkeypatch): assert decrypt_value_helper(legacy_v2, key="t") == "v2-secret" -def test_legacy_ciphertext_without_pynacl_logs_the_reencrypt_path_and_keeps_the_stored_value(monkeypatch, caplog): +def test_legacy_ciphertext_without_pynacl_logs_the_reencrypt_path_and_never_returns_the_blob(monkeypatch, caplog): legacy = _legacy_nacl_ciphertext("legacy-secret", _sha256_key()) assert legacy_encryption_available() monkeypatch.setitem(sys.modules, "nacl", None) @@ -185,7 +186,7 @@ def test_legacy_ciphertext_without_pynacl_logs_the_reencrypt_path_and_keeps_the_ assert not legacy_encryption_available() with caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): - assert decrypt_value_helper(legacy, key="t", exception_type="debug", return_original_value=True) == legacy + assert decrypt_value_helper(legacy, key="t", exception_type="debug", return_original_value=True) is None assert decrypt_value_helper(legacy, key="t", exception_type="debug") is None assert decrypt_if_encrypted_with(legacy, _SALT_KEY) is None assert len(caplog.records) == 3, caplog.text @@ -197,6 +198,15 @@ def test_legacy_ciphertext_without_pynacl_logs_the_reencrypt_path_and_keeps_the_ assert "legacy-secret" not in caplog.text +def test_require_legacy_reader_refuses_rewrite_passes_without_pynacl(monkeypatch): + require_legacy_reader("rotate the master key") + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + with pytest.raises(LegacyEncryptionUnavailableError, match="rotate the master key.*legacy-encryption"): + require_legacy_reader("rotate the master key") + + def test_legacy_opt_in_without_pynacl_fails_the_write_not_silently(monkeypatch): _use_legacy(monkeypatch) monkeypatch.setitem(sys.modules, "nacl", None) diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 3b86f1f6d20..0738441c59c 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -9435,6 +9435,36 @@ async def test_rotate_master_key_reencrypts_model_params_in_place( ), "api_key must be stored re-encrypted under the new master key, not in plaintext" +async def test_rotate_master_key_refuses_without_pynacl_before_touching_any_row(monkeypatch): + import sys + from unittest.mock import AsyncMock, MagicMock + + from fastapi import HTTPException + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _rotate_master_key, + ) + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + mock_prisma_client = AsyncMock() + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + + with pytest.raises(HTTPException) as exc_info: + await _rotate_master_key( + prisma_client=mock_prisma_client, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test-user"), + current_master_key="sk-old-master-key", + new_master_key="sk-new-master-key", + ) + + assert exc_info.value.status_code == 400 + assert "legacy-encryption" in exc_info.value.detail["error"] + mock_prisma_client.db.litellm_proxymodeltable.find_many.assert_not_awaited() + + async def test_default_key_generate_params_duration(monkeypatch): """ Test that default_key_generate_params with 'duration' is applied diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 7e198bc9131..cf3676076e8 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -13,6 +13,7 @@ import json import logging import os import re +import sys from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime @@ -22,10 +23,14 @@ from typing import Any, Dict, Final from unittest.mock import AsyncMock, MagicMock import pytest +from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm.proxy._types import CommonProxyErrors -from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + LegacyEncryptionUnavailableError, + encrypt_value_helper, +) from litellm.proxy.proxy_server import ( ProxyConfig, _is_remote_module_url, @@ -33,14 +38,13 @@ from litellm.proxy.proxy_server import ( _scrub_guardrail_inner, resolve_complexity_router_plugins, resolve_routing_plugins, + validate_auto_router_capability_limits, validate_deployment_access_windows, validate_deployment_complexity_router_placement, validate_deployment_max_agentic_loops, - validate_auto_router_capability_limits, ) from .conftest import normalize -from pydantic import JsonValue, TypeAdapter, ValidationError # --------------------------------------------------------------------------- # _is_remote_module_url @@ -3499,6 +3503,16 @@ def test_ProxyConfig__encrypt_env_variables_for_db_idempotent(monkeypatch): assert out == {"A": "ENC[1]", "B": "ENC[2]", "C": "ENC[3]"} +def test_ProxyConfig__encrypt_env_variables_for_db_refuses_without_pynacl_instead_of_dropping_legacy_values( + monkeypatch, +): + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + pc = ProxyConfig() + with pytest.raises(LegacyEncryptionUnavailableError, match="config save.*legacy-encryption"): + pc._encrypt_env_variables_for_db({"A": "1"}) + + def test_ProxyConfig__encrypt_env_variables_for_db_invalid_raises(): pc = ProxyConfig() with pytest.raises(AttributeError): From 9a1434e6ff766f615661bf8ddf0b6fda2130f1fa Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 11:14:34 +0000 Subject: [PATCH 10/17] test(proxy): use raw strings for regex match patterns Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/common_utils/test_encrypt_decrypt_utils.py | 2 +- tests/test_litellm/proxy/proxy_server/test_proxy_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py index 6c0118576ea..cf9ad5b1c1e 100644 --- a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -203,7 +203,7 @@ def test_require_legacy_reader_refuses_rewrite_passes_without_pynacl(monkeypatch monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) - with pytest.raises(LegacyEncryptionUnavailableError, match="rotate the master key.*legacy-encryption"): + with pytest.raises(LegacyEncryptionUnavailableError, match=r"rotate the master key.*legacy-encryption"): require_legacy_reader("rotate the master key") diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index cf3676076e8..b50dbc17e21 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -3509,7 +3509,7 @@ def test_ProxyConfig__encrypt_env_variables_for_db_refuses_without_pynacl_instea monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) pc = ProxyConfig() - with pytest.raises(LegacyEncryptionUnavailableError, match="config save.*legacy-encryption"): + with pytest.raises(LegacyEncryptionUnavailableError, match=r"config save.*legacy-encryption"): pc._encrypt_env_variables_for_db({"A": "1"}) From 6179f8e5d97427d312009b52d307b4aedb0d761a Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 11:21:04 +0000 Subject: [PATCH 11/17] fix(proxy): mark the master key rotation error detail as a FastAPI detail contract Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/management_endpoints/key_management_endpoints.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index d2488c08e6a..68a2c3fa7d7 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -5182,7 +5182,10 @@ async def _rotate_master_key( try: require_legacy_reader("rotate the master key") except LegacyEncryptionUnavailableError as error: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail={"error": str(error)}) from error + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": str(error)}, # mutable-ok: FastAPI detail contract + ) from error try: models: list | None = cast( # cast-ok: find_many returns a real list, which TableActions widens to Sequence From 494f423c9e2a73994ff1ed0e82a049542bdffb3c Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 11:59:12 +0000 Subject: [PATCH 12/17] fix(proxy): require PyNaCl only when an unprefixed stored value must be read Config save, migration, scan and master key rotation now pass on plaintext or v3 only stores without PyNaCl and refuse only when a non empty unprefixed value is met Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../common_utils/encrypt_decrypt_utils.py | 23 ++++++++++++------- .../credential_migration.py | 6 ++--- .../key_management_endpoints.py | 14 +++++++++-- litellm/proxy/proxy_server.py | 6 +++-- .../test_encrypt_decrypt_utils.py | 14 +++++++---- .../test_credential_migration.py | 22 ++++++++++++++---- .../test_key_management_endpoints.py | 22 ++++++++++++++++-- .../proxy/proxy_server/test_proxy_config.py | 9 +++++++- 8 files changed, 89 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 444a8590ef4..8b32b7348b7 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -1,7 +1,7 @@ import base64 import hashlib import os -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Final, Literal, cast from pydantic import TypeAdapter, ValidationError @@ -221,13 +221,20 @@ def legacy_encryption_available() -> bool: return True -def require_legacy_reader(purpose: str) -> None: - """Refuse a decrypt-then-rewrite pass when PyNaCl is missing: every unprefixed value would read as - unreadable and be dropped, double wrapped or miscounted as plaintext, so the pass cannot be trusted.""" - if not legacy_encryption_available(): - raise LegacyEncryptionUnavailableError( - f"Cannot {purpose}: PyNaCl is needed to read legacy {_ALGO_XSALSA20} values. {_LEGACY_ENCRYPTION_HELP}" - ) +def needs_legacy_reader(value: object) -> bool: + """True for a stored string that only PyNaCl can tell apart from plaintext: non empty and unprefixed.""" + return isinstance(value, str) and value != "" and not is_versioned_gcm(value) + + +def require_legacy_reader_for(values: Iterable[object], purpose: str) -> None: + """Refuse a decrypt-then-rewrite pass when PyNaCl is missing and one of the values is unprefixed: it would + read as unreadable and be dropped, double wrapped or miscounted as plaintext. Versioned gcm and non string + values never need PyNaCl, so a fully migrated store passes.""" + if legacy_encryption_available() or not any(needs_legacy_reader(value) for value in values): + return + raise LegacyEncryptionUnavailableError( + f"Cannot {purpose}: PyNaCl is needed to read legacy {_ALGO_XSALSA20} values. {_LEGACY_ENCRYPTION_HELP}" + ) def _legacy_secret_box(signing_key: str, purpose: str) -> "SecretBox": diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index 0cbdb4f0a78..2e98364c7db 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -49,7 +49,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, is_versioned_gcm, - require_legacy_reader, + require_legacy_reader_for, ) ValueClass = Literal["migrated", "legacy", "plaintext", "undecryptable", "not-a-string"] @@ -148,6 +148,7 @@ def classify_value(value: object, key: str = "scan") -> ValueClass: return "plaintext" if is_versioned_gcm(value): return "migrated" + require_legacy_reader_for((value,), "scan stored encryption") decrypted: Final = decrypt_value_helper(value=value, key=key, exception_type="debug", return_original_value=False) if decrypted is None: # Did not decrypt under nacl and has no v2 marker: legacy plaintext. @@ -167,6 +168,7 @@ def reencrypt_value(value: object, key: str = "migrate") -> object: return value if is_versioned_gcm(value): return value # idempotent: already migrated + require_legacy_reader_for((value,), "migrate stored encryption") decrypted: Final = decrypt_value_helper(value=value, key=key, exception_type="debug", return_original_value=False) if decrypted is None: # Either legacy plaintext (no ciphertext to migrate) or corrupt. Either @@ -634,7 +636,6 @@ async def migrate_encryption( net-new walkers run in dry-run mode. """ _assert_aes_gate_enabled() - require_legacy_reader("migrate stored encryption") report: Final = MigrationReport() @@ -667,7 +668,6 @@ async def check_encryption(prisma_client: object) -> MigrationReport: config rows, SSO config). Reports how many values are still ``legacy``; ``residual_legacy == 0`` across this full scan is the compliance attestation. """ - require_legacy_reader("scan stored encryption") report: Final = MigrationReport() # Rotation-covered tables (read-only classification). diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 68a2c3fa7d7..6cc45489fa3 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -81,7 +81,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( ) from litellm.proxy.common_utils.encrypt_decrypt_utils import ( LegacyEncryptionUnavailableError, - require_legacy_reader, + legacy_encryption_available, ) from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time @@ -5156,6 +5156,16 @@ async def delete_key_aliases( ) +async def _require_legacy_reader_for_stored_values(prisma_client: PrismaClient) -> None: + """Without PyNaCl, scan the rotation covered tables read only so an unprefixed stored value refuses the + rotation before any row is rewritten. A fully migrated store passes; with PyNaCl installed nothing runs""" + if legacy_encryption_available(): + return + from litellm.proxy.management_endpoints.credential_migration import _scan_covered_tables + + await _scan_covered_tables(prisma_client) + + async def _rotate_master_key( prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth, @@ -5180,7 +5190,7 @@ async def _rotate_master_key( from litellm.proxy.proxy_server import proxy_config try: - require_legacy_reader("rotate the master key") + await _require_legacy_reader_for_stored_values(prisma_client) except LegacyEncryptionUnavailableError as error: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e2524148597..b460b1bff91 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -403,7 +403,7 @@ from litellm.proxy.common_utils.discoverable_model_filter import discoverable_ro from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, - require_legacy_reader, + require_legacy_reader_for, ) from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id from litellm.proxy.common_utils.fips import ( @@ -7326,7 +7326,9 @@ class ProxyConfig: _decrypt_and_set_db_env_variables): this is a write path, and loading values into os.environ is the read path's responsibility. """ - require_legacy_reader("re-encrypt environment variables for the config save") + require_legacy_reader_for( + environment_variables.values(), "re-encrypt environment variables for the config save" + ) decrypted_env_vars: Final = self._decrypt_db_variables(environment_variables) return self._encrypt_env_variables( environment_variables=decrypted_env_vars, diff --git a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py index cf9ad5b1c1e..229f46630f9 100644 --- a/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -29,7 +29,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, is_versioned_gcm, legacy_encryption_available, - require_legacy_reader, + require_legacy_reader_for, ) from litellm.proxy.common_utils.fips import FipsModeError @@ -198,13 +198,17 @@ def test_legacy_ciphertext_without_pynacl_logs_the_reencrypt_path_and_never_retu assert "legacy-secret" not in caplog.text -def test_require_legacy_reader_refuses_rewrite_passes_without_pynacl(monkeypatch): - require_legacy_reader("rotate the master key") +def test_require_legacy_reader_for_refuses_only_unprefixed_values_without_pynacl(monkeypatch): + legacy = _legacy_nacl_ciphertext("legacy-secret", _sha256_key()) + v3 = encrypt_value_helper("v3-secret") + require_legacy_reader_for((legacy, v3, "plain"), "rotate the master key") monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) - with pytest.raises(LegacyEncryptionUnavailableError, match=r"rotate the master key.*legacy-encryption"): - require_legacy_reader("rotate the master key") + require_legacy_reader_for((v3, "", None, 3), "rotate the master key") + for unreadable in (legacy, "plain-or-legacy"): + with pytest.raises(LegacyEncryptionUnavailableError, match=r"rotate the master key.*legacy-encryption"): + require_legacy_reader_for((v3, unreadable), "rotate the master key") def test_legacy_opt_in_without_pynacl_fails_the_write_not_silently(monkeypatch): diff --git a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py index 4c4ee2e95de..eb00462141e 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py +++ b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py @@ -130,17 +130,31 @@ async def test_migrate_requires_aes_gate(salt_key, monkeypatch): @pytest.mark.asyncio async def test_migrate_and_check_refuse_without_pynacl_instead_of_miscounting_legacy_rows(salt_key, monkeypatch): + legacy: Final = _legacy_ct("model-secret", monkeypatch) + _enable_aes(monkeypatch) monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) client: Final = MagicMock() _empty_covered_tables(client) + client.db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + client.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) + client.db.litellm_config.find_unique = AsyncMock(return_value=None) + client.db.litellm_config.update = AsyncMock() - with pytest.raises(RuntimeError, match="legacy-encryption"): - await cm.migrate_encryption(prisma_client=client, user_api_key_dict=MagicMock()) + v3_only: Final = await cm.check_encryption(prisma_client=client) + assert v3_only.residual_legacy == 0 + + client.db.litellm_proxymodeltable.find_many = AsyncMock( + return_value=[SimpleNamespace(litellm_params={"api_key": legacy})] + ) + client.db.litellm_proxymodeltable.update_many = AsyncMock() with pytest.raises(RuntimeError, match="legacy-encryption"): await cm.check_encryption(prisma_client=client) - for _, db_attr, _, _ in cm._COVERED_TABLE_SPECS: - getattr(client.db, db_attr).find_many.assert_not_awaited() + with pytest.raises(RuntimeError, match="legacy-encryption"): + await cm.migrate_encryption(prisma_client=client, user_api_key_dict=MagicMock()) + client.db.litellm_proxymodeltable.update_many.assert_not_awaited() + client.db.litellm_config.update.assert_not_awaited() # --------------------------- config-row walker --------------------------- diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 0738441c59c..46602c10888 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -9437,20 +9437,37 @@ async def test_rotate_master_key_reencrypts_model_params_in_place( async def test_rotate_master_key_refuses_without_pynacl_before_touching_any_row(monkeypatch): import sys + from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock from fastapi import HTTPException + from litellm.proxy import proxy_server from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.management_endpoints.key_management_endpoints import ( _rotate_master_key, ) + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-rotate") + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) + legacy_row = SimpleNamespace(litellm_params={"api_key": encrypt_value_helper("legacy-secret")}) + monkeypatch.setattr(proxy_server, "general_settings", {}) monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) mock_prisma_client = AsyncMock() mock_prisma_client.db = MagicMock() - mock_prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[legacy_row]) + mock_prisma_client.db.litellm_proxymodeltable.update_many = AsyncMock() + mock_prisma_client.db.litellm_config.find_unique = AsyncMock(return_value=None) + empty_tables = ( + "litellm_credentialstable", + "litellm_mcpservertable", + "litellm_mcpusercredentials", + "litellm_mcpuserenvvars", + ) + for table in empty_tables: + getattr(mock_prisma_client.db, table).find_many = AsyncMock(return_value=[]) with pytest.raises(HTTPException) as exc_info: await _rotate_master_key( @@ -9462,7 +9479,8 @@ async def test_rotate_master_key_refuses_without_pynacl_before_touching_any_row( assert exc_info.value.status_code == 400 assert "legacy-encryption" in exc_info.value.detail["error"] - mock_prisma_client.db.litellm_proxymodeltable.find_many.assert_not_awaited() + mock_prisma_client.db.litellm_proxymodeltable.update_many.assert_not_awaited() + mock_prisma_client.db.tx.assert_not_called() async def test_default_key_generate_params_duration(monkeypatch): diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index b50dbc17e21..ac4993a4327 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -29,6 +29,7 @@ import litellm from litellm.proxy._types import CommonProxyErrors from litellm.proxy.common_utils.encrypt_decrypt_utils import ( LegacyEncryptionUnavailableError, + decrypt_value_helper, encrypt_value_helper, ) from litellm.proxy.proxy_server import ( @@ -3506,11 +3507,17 @@ def test_ProxyConfig__encrypt_env_variables_for_db_idempotent(monkeypatch): def test_ProxyConfig__encrypt_env_variables_for_db_refuses_without_pynacl_instead_of_dropping_legacy_values( monkeypatch, ): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-config-save") + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + v3 = encrypt_value_helper("already-migrated") monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) pc = ProxyConfig() + + saved = pc._encrypt_env_variables_for_db({"A": v3}) + assert decrypt_value_helper(saved["A"], key="A") == "already-migrated" with pytest.raises(LegacyEncryptionUnavailableError, match=r"config save.*legacy-encryption"): - pc._encrypt_env_variables_for_db({"A": "1"}) + pc._encrypt_env_variables_for_db({"A": v3, "B": "plain-or-legacy"}) def test_ProxyConfig__encrypt_env_variables_for_db_invalid_raises(): From 33834c679b6b9d2ec53152efcd3105fcec1055e5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 12:22:58 +0000 Subject: [PATCH 13/17] fix(proxy): cover MCP OAuth clients, SSO assertions and MCP secret maps in the missing PyNaCl preflight Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../credential_migration.py | 17 ++++++++- .../test_credential_migration.py | 36 +++++++++++++++++++ .../test_key_management_endpoints.py | 31 +++++++++++----- 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index 2e98364c7db..db6e518a663 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -462,6 +462,8 @@ _COVERED_TABLE_SPECS: Final = [ ("mcp_server", "litellm_mcpservertable", ("credentials", "env_vars", "static_headers", "env"), ()), ("mcp_user_credentials", "litellm_mcpusercredentials", (), ("credential_b64",)), ("mcp_user_env_vars", "litellm_mcpuserenvvars", (), ("values_b64",)), + ("mcp_oauth_client", "litellm_mcpserveroauthclient", ("credentials",), ()), + ("sso_identity_assertion", "litellm_ssoidentityassertion", (), ("assertion_b64",)), ] @@ -493,14 +495,27 @@ def _classify_into_report(report: LocationReport, value: str) -> None: report.count(classify_value(value, key="scan")) +def _secret_map_ciphertext(value: object) -> object: + """The stored ciphertext behind an MCP secret map: the JSON quoted string is unwrapped, a plain map or + anything that is not JSON is returned as is for ``decode_secret_map`` to judge.""" + if not isinstance(value, str) or not value.lstrip().startswith('"'): + return value + try: + return json.loads(value) + except ValueError: + return value + + def _classify_secret_map(value: object, key: str) -> ValueClass | None: + ciphertext: Final = _secret_map_ciphertext(value) + if isinstance(ciphertext, str) and not ciphertext.lstrip().startswith("{"): + require_legacy_reader_for((ciphertext,), "scan stored encryption") try: decoded: Final = decode_secret_map(value, key=key) except SecretMapDecodeError: return "undecryptable" if not decoded: return None - ciphertext: Final = json.loads(value) if isinstance(value, str) and value.lstrip().startswith('"') else value return "migrated" if is_migrated(ciphertext) else "legacy" diff --git a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py index eb00462141e..24eb3cdab78 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py +++ b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py @@ -17,6 +17,7 @@ import pytest from litellm.proxy import proxy_server from litellm.proxy.common_utils.encrypt_decrypt_utils import ( _V3_GCM_PREFIX, + encrypt_secret_map, encrypt_value_helper, ) from litellm.proxy.management_endpoints import credential_migration as cm @@ -157,6 +158,41 @@ async def test_migrate_and_check_refuse_without_pynacl_instead_of_miscounting_le client.db.litellm_config.update.assert_not_awaited() +@pytest.mark.parametrize( + "db_attr, build_row", + [ + ("litellm_mcpservertable", lambda ct: SimpleNamespace(server_id="s1", static_headers=json.dumps(ct))), + ("litellm_mcpservertable", lambda ct: SimpleNamespace(server_id="s1", env=ct)), + ("litellm_mcpserveroauthclient", lambda ct: SimpleNamespace(server_id="s1", credentials={"client_secret": ct})), + ("litellm_ssoidentityassertion", lambda ct: SimpleNamespace(user_id="u1", assertion_b64=ct)), + ], +) +@pytest.mark.asyncio +async def test_check_refuses_without_pynacl_for_every_rotation_rewritten_location( + db_attr, build_row, salt_key, monkeypatch +): + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) + legacy_map: Final = json.loads(encrypt_secret_map({"Authorization": "Bearer legacy"})) + legacy_value: Final = encrypt_value_helper("legacy-secret") + _enable_aes(monkeypatch) + client: Final = MagicMock() + _empty_covered_tables(client) + client.db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + client.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) + client.db.litellm_config.find_unique = AsyncMock(return_value=None) + ciphertext: Final = legacy_map if db_attr == "litellm_mcpservertable" else legacy_value + getattr(client.db, db_attr).find_many = AsyncMock(return_value=[build_row(ciphertext)]) + + with_pynacl: Final = await cm.check_encryption(prisma_client=client) + assert with_pynacl.residual_legacy == 1, with_pynacl.to_dict() + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + with pytest.raises(RuntimeError, match="legacy-encryption"): + await cm.check_encryption(prisma_client=client) + + # --------------------------- config-row walker --------------------------- diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 46602c10888..882e38343a6 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -9435,9 +9435,16 @@ async def test_rotate_master_key_reencrypts_model_params_in_place( ), "api_key must be stored re-encrypted under the new master key, not in plaintext" -async def test_rotate_master_key_refuses_without_pynacl_before_touching_any_row(monkeypatch): +@pytest.mark.parametrize( + "legacy_table, build_row", + [ + ("litellm_proxymodeltable", lambda ct: SimpleNamespace(litellm_params={"api_key": ct})), + ("litellm_mcpserveroauthclient", lambda ct: SimpleNamespace(server_id="s1", credentials={"client_secret": ct})), + ("litellm_ssoidentityassertion", lambda ct: SimpleNamespace(user_id="u1", assertion_b64=ct)), + ], +) +async def test_rotate_master_key_refuses_without_pynacl_before_touching_any_row(legacy_table, build_row, monkeypatch): import sys - from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock from fastapi import HTTPException @@ -9451,23 +9458,27 @@ async def test_rotate_master_key_refuses_without_pynacl_before_touching_any_row( monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-rotate") monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) - legacy_row = SimpleNamespace(litellm_params={"api_key": encrypt_value_helper("legacy-secret")}) + legacy_row = build_row(encrypt_value_helper("legacy-secret")) monkeypatch.setattr(proxy_server, "general_settings", {}) monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl.secret", None) mock_prisma_client = AsyncMock() mock_prisma_client.db = MagicMock() - mock_prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=[legacy_row]) - mock_prisma_client.db.litellm_proxymodeltable.update_many = AsyncMock() mock_prisma_client.db.litellm_config.find_unique = AsyncMock(return_value=None) - empty_tables = ( + covered_tables = ( + "litellm_proxymodeltable", "litellm_credentialstable", "litellm_mcpservertable", "litellm_mcpusercredentials", "litellm_mcpuserenvvars", + "litellm_mcpserveroauthclient", + "litellm_ssoidentityassertion", ) - for table in empty_tables: - getattr(mock_prisma_client.db, table).find_many = AsyncMock(return_value=[]) + for table in covered_tables: + rows = [legacy_row] if table == legacy_table else [] + getattr(mock_prisma_client.db, table).find_many = AsyncMock(return_value=rows) + getattr(mock_prisma_client.db, table).update = AsyncMock() + getattr(mock_prisma_client.db, table).update_many = AsyncMock() with pytest.raises(HTTPException) as exc_info: await _rotate_master_key( @@ -9479,7 +9490,9 @@ async def test_rotate_master_key_refuses_without_pynacl_before_touching_any_row( assert exc_info.value.status_code == 400 assert "legacy-encryption" in exc_info.value.detail["error"] - mock_prisma_client.db.litellm_proxymodeltable.update_many.assert_not_awaited() + for table in covered_tables: + getattr(mock_prisma_client.db, table).update.assert_not_awaited() + getattr(mock_prisma_client.db, table).update_many.assert_not_awaited() mock_prisma_client.db.tx.assert_not_called() From f162996af31379dc75b8b41bc3b5a14b845e61d2 Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 12:46:13 +0000 Subject: [PATCH 14/17] fix(proxy): scope the PyNaCl-free encryption scan to MCP fields encrypted at rest Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_experimental/mcp_server/db.py | 22 +++++----- .../credential_migration.py | 29 ++++++++++++- .../test_credential_migration.py | 43 +++++++++++++++++++ .../test_key_management_endpoints.py | 43 +++++++++++++++++++ 4 files changed, 126 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 1ec0f24ae38..28ba5d32bb7 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -714,20 +714,22 @@ async def _db_find_user_env_var_rows( return await _user_env_var_actions(prisma_client).find_many(where=where) +MCP_CREDENTIAL_SECRET_FIELDS: Final = ( + "auth_value", + "client_id", + "client_secret", + "client_private_key", + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", +) + + def decrypt_credentials( credentials: MCPCredentials, ) -> MCPCredentials: """Decrypt all secret fields in an MCPCredentials dict using the global salt key.""" - secret_fields: Final = [ - "auth_value", - "client_id", - "client_secret", - "client_private_key", - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", - ] - for field in secret_fields: + for field in MCP_CREDENTIAL_SECRET_FIELDS: value = credentials.get(field) if value is not None and isinstance(value, str): credentials[field] = decrypt_value_helper( diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index db6e518a663..61dd636e0dd 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -31,6 +31,7 @@ config rows, and the SSO config table. """ import json +from collections.abc import Iterator from dataclasses import dataclass, field from typing import TYPE_CHECKING, Final, Literal, cast @@ -465,6 +466,7 @@ _COVERED_TABLE_SPECS: Final = [ ("mcp_oauth_client", "litellm_mcpserveroauthclient", ("credentials",), ()), ("sso_identity_assertion", "litellm_ssoidentityassertion", (), ("assertion_b64",)), ] +_MCP_CREDENTIAL_TABLES: Final = frozenset({"litellm_mcpservertable", "litellm_mcpserveroauthclient"}) def _iter_encrypted_strings(obj: object): @@ -485,6 +487,31 @@ def _iter_encrypted_strings(obj: object): stack.extend(cur) +def _mcp_encrypted_leaves(col: str, raw: object) -> Iterator[str]: + """Only the strings an MCP column encrypts at rest: a credentials blob keeps auth_type, scopes and urls in + plaintext and env_vars keeps every name and every per user placeholder, so without PyNaCl those must not be + mistaken for legacy ciphertext and refuse the scan""" + from litellm.proxy._experimental.mcp_server.db import MCP_CREDENTIAL_SECRET_FIELDS, _is_global_env_var_scope + + if col == "credentials": + if not isinstance(raw, dict): + return iter(()) + return (v for k, v in raw.items() if k in MCP_CREDENTIAL_SECRET_FIELDS and isinstance(v, str)) + if not isinstance(raw, list): + return iter(()) + return ( + e["value"] + for e in raw + if isinstance(e, dict) and _is_global_env_var_scope(e.get("scope")) and isinstance(e.get("value"), str) + ) + + +def _encrypted_leaves(db_attr: str, col: str, raw: object) -> Iterator[str]: + if db_attr in _MCP_CREDENTIAL_TABLES and col in ("credentials", "env_vars"): + return _mcp_encrypted_leaves(col, raw) + return _iter_encrypted_strings(raw) + + def _classify_into_report(report: LocationReport, value: str) -> None: """Classify one stored string and bump the matching read-only counter. @@ -548,7 +575,7 @@ async def _scan_one_table( raw = json.loads(raw) except (ValueError, TypeError): pass - for s in _iter_encrypted_strings(raw): + for s in _encrypted_leaves(db_attr, col, raw): _classify_into_report(report, s) for col in scalar_columns: v = getattr(row, col, None) diff --git a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py index 24eb3cdab78..b6782ec602d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py +++ b/tests/test_litellm/proxy/management_endpoints/test_credential_migration.py @@ -193,6 +193,49 @@ async def test_check_refuses_without_pynacl_for_every_rotation_rewritten_locatio await cm.check_encryption(prisma_client=client) +@pytest.mark.asyncio +async def test_check_without_pynacl_ignores_plaintext_mcp_metadata_next_to_v3_secrets(salt_key, monkeypatch): + _enable_aes(monkeypatch) + v3_secret: Final = encrypt_value_helper("dcr-secret") + client: Final = MagicMock() + _empty_covered_tables(client) + client.db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + client.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=None) + client.db.litellm_config.find_unique = AsyncMock(return_value=None) + client.db.litellm_mcpserveroauthclient.find_many = AsyncMock( + return_value=[ + SimpleNamespace( + server_id="s1", + credentials={"client_id": v3_secret, "client_secret": v3_secret, "scopes": ["a"], "auth_type": "oauth2"}, + ) + ] + ) + client.db.litellm_mcpservertable.find_many = AsyncMock( + return_value=[ + SimpleNamespace( + server_id="s1", + credentials={"auth_value": v3_secret, "auth_type": "api_key", "token_url": "https://idp/token"}, + env_vars=[ + {"name": "TOKEN", "scope": "global", "value": v3_secret}, + {"name": "USER_TOKEN", "scope": "user", "value": "{{user.token}}"}, + ], + ) + ] + ) + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + report: Final = await cm.check_encryption(prisma_client=client) + + by_location: Final = report.as_dict()["locations"] + assert report.residual_legacy == 0, by_location + assert by_location["mcp_oauth_client"]["already_v2"] == 2, by_location + assert by_location["mcp_oauth_client"]["scanned"] == 2, by_location + assert by_location["mcp_server"]["already_v2"] == 2, by_location + assert by_location["mcp_server"]["scanned"] == 2, by_location + + # --------------------------- config-row walker --------------------------- diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index 882e38343a6..eadf278a1ca 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -9496,6 +9496,49 @@ async def test_rotate_master_key_refuses_without_pynacl_before_touching_any_row( mock_prisma_client.db.tx.assert_not_called() +async def test_rotate_master_key_preflight_without_pynacl_passes_migrated_oauth_client_with_plaintext_scopes( + monkeypatch, +): + import sys + from unittest.mock import AsyncMock, MagicMock + + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _require_legacy_reader_for_stored_values, + ) + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-rotate") + monkeypatch.setattr(proxy_server, "general_settings", {}) + v3_secret = encrypt_value_helper("dcr-secret") + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + mock_prisma_client = AsyncMock() + mock_prisma_client.db = MagicMock() + mock_prisma_client.db.litellm_config.find_unique = AsyncMock(return_value=None) + for table in ( + "litellm_proxymodeltable", + "litellm_credentialstable", + "litellm_mcpservertable", + "litellm_mcpusercredentials", + "litellm_mcpuserenvvars", + "litellm_ssoidentityassertion", + ): + getattr(mock_prisma_client.db, table).find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.litellm_mcpserveroauthclient.find_many = AsyncMock( + return_value=[ + SimpleNamespace( + server_id="s1", + credentials={"client_id": v3_secret, "client_secret": v3_secret, "scopes": ["a"], "auth_type": "oauth2"}, + ) + ] + ) + + await _require_legacy_reader_for_stored_values(mock_prisma_client) + + assert mock_prisma_client.db.litellm_mcpserveroauthclient.find_many.await_count == 1 + + async def test_default_key_generate_params_duration(monkeypatch): """ Test that default_key_generate_params with 'duration' is applied From b21ed6816cb26eb5e28ea5926e8ffaeb8630b20f Mon Sep 17 00:00:00 2001 From: yucheng Date: Thu, 24 Sep 2026 12:53:06 +0000 Subject: [PATCH 15/17] refactor(proxy): drop routine source comment on the encryption algorithm setting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_utils/encrypt_decrypt_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 8b32b7348b7..8e4e1237a74 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -23,8 +23,6 @@ _V2_GCM_PREFIX: Final = "v2:gcm:" _GCM_PREFIXES: Final = (_V3_GCM_PREFIX, _V2_GCM_PREFIX) _HKDF_INFO: Final = b"litellm-at-rest-v3" -# general_settings key selecting the at-rest encryption algorithm for new writes. -# Decrypt is always format-detecting, so the setting never strands stored data. _ENCRYPTION_ALGORITHM_SETTING: Final = "encryption_algorithm" _ALGO_AES_GCM: Final = "aes-256-gcm" _ALGO_XSALSA20: Final = "xsalsa20-poly1305" From 20dea51f28398efb328d4f7a24a277abd395fce8 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 5 Oct 2026 08:33:46 +0000 Subject: [PATCH 16/17] fix(proxy): drop unreadable legacy credential values instead of serving the ciphertext and restore the non-root image from main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- docker/Dockerfile.non_root | 38 ++++++------------- litellm/proxy/_experimental/mcp_server/db.py | 8 ++-- .../common_utils/credential_hydration.py | 22 +++++++---- .../common_utils/encrypt_decrypt_utils.py | 6 +++ .../credential_migration.py | 14 +++---- .../key_management_endpoints.py | 6 +-- litellm/proxy/proxy_server.py | 8 +--- .../common_utils/test_credential_hydration.py | 32 ++++++++++++++++ .../test_credential_migration.py | 4 +- .../proxy/proxy_server/test_proxy_config.py | 22 +++++++++++ 10 files changed, 104 insertions(+), 56 deletions(-) diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 364a763c7fa..6bf74c294fc 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -3,7 +3,6 @@ # Base images ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -44,7 +43,6 @@ COPY ui/litellm-dashboard/ ./ RUN npm run build FROM $LITELLM_BUILD_IMAGE AS builder -ARG PROXY_EXTRAS_SOURCE WORKDIR /app USER root @@ -104,32 +102,19 @@ ENV LITELLM_NON_ROOT=true RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \ cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \ - cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \ + cp /app/litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \ touch /var/lib/litellm/ui/.litellm_ui_ready RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \ - if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \ - uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra legacy-encryption \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13 \ - --no-sources-package litellm-proxy-extras; \ - else \ - uv sync --frozen --no-default-groups --no-editable \ - --extra proxy \ - --extra legacy-encryption \ - --extra proxy-runtime \ - --extra extra_proxy \ - --extra semantic-router \ - --extra saml \ - --extra bedrock-realtime \ - --python python3.13; \ - fi + uv sync --frozen --no-default-groups --no-editable \ + --extra proxy \ + --extra legacy-encryption \ + --extra proxy-runtime \ + --extra extra_proxy \ + --extra semantic-router \ + --extra saml \ + --extra bedrock-realtime \ + --python python3.13 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \ npm_config_cache=/root/.npm \ @@ -139,7 +124,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh FROM $LITELLM_RUNTIME_IMAGE AS runtime -ARG PROXY_EXTRAS_SOURCE +ARG LITELLM_RELEASE_TAG="" +ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} WORKDIR /app USER root diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index 5b8469d4209..503a0c722f8 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -180,7 +180,7 @@ def _drop_stale_minted_on_client_rotation(merged: dict[str, object], new_creds: } -def _is_global_env_var_scope(scope: object) -> bool: +def is_global_env_var_scope(scope: object) -> bool: """``scope="user"`` entries are placeholders the user fills in; everything else (including a missing scope) is an admin-supplied global value.""" return scope != MCPEnvVarScope.user and scope != "user" @@ -195,7 +195,7 @@ def _encrypt_global_env_var_values(env_vars: Iterable[dict[str, str]]) -> None: secrets and are stored verbatim. """ for entry in env_vars: - if not _is_global_env_var_scope(entry.get("scope")): + if not is_global_env_var_scope(entry.get("scope")): continue value = entry.get("value") if value: @@ -216,7 +216,7 @@ def decrypt_global_env_var_values(env_vars: Iterable[MCPEnvVar | dict[str, str]] for entry in env_vars: is_dict = isinstance(entry, dict) scope = entry.get("scope") if is_dict else getattr(entry, "scope", None) - if not _is_global_env_var_scope(scope): + if not is_global_env_var_scope(scope): continue value = entry.get("value") if is_dict else getattr(entry, "value", None) if not value: @@ -296,7 +296,7 @@ def _reencrypt_global_env_var_values( rebuilt: Final = [dict(v) for v in entries] rotated = False for entry in rebuilt: - if not _is_global_env_var_scope(entry.get("scope")): + if not is_global_env_var_scope(entry.get("scope")): continue value = entry.get("value") if not value: diff --git a/litellm/proxy/common_utils/credential_hydration.py b/litellm/proxy/common_utils/credential_hydration.py index 2aabe8cad5c..7f4819b49fb 100644 --- a/litellm/proxy/common_utils/credential_hydration.py +++ b/litellm/proxy/common_utils/credential_hydration.py @@ -12,7 +12,7 @@ from types import MappingProxyType from typing import Final import litellm -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, legacy_unreadable from litellm.proxy.utils import PrismaClient from litellm.repositories.credentials_repository import CredentialsRepository from litellm.router_utils.clientside_credential_handler import clientside_credential_keys @@ -58,20 +58,26 @@ def stored_credential_provider(credential_provider: object) -> str | None: return lowered if lowered in _LITELLM_PROVIDER_IDS else None -def decrypted_or_stored(key: str, value: str) -> str: - """The stored value decrypted, or as stored when it was never encrypted (a config.yaml value).""" +def decrypted_or_stored(key: str, value: str) -> str | None: + """The stored value decrypted, as stored when it was never encrypted (a config.yaml value), or None when it + is legacy ciphertext this install has no PyNaCl to read: a credential, never the ciphertext blob.""" decrypted: Final = decrypt_value_helper(value=value, key=key) - return value if decrypted is None else decrypted + if decrypted is not None: + return decrypted + return None if legacy_unreadable(value) else value + + +def decrypted_values(values: Mapping[str, str]) -> Mapping[str, str]: + """``values`` with every entry decrypted, plaintext entries kept, and unreadable legacy entries dropped.""" + resolved: Final = {key: decrypted_or_stored(key, value) for key, value in values.items()} + return MappingProxyType({key: value for key, value in resolved.items() if value is not None}) def _decrypted(db_credential: CredentialItem) -> CredentialItem: """The stored credential with every value decrypted, leaving already-plaintext values alone.""" - decrypted_values: Final = MappingProxyType( - {key: decrypted_or_stored(key, value) for key, value in db_credential.credential_values.items()} - ) return CredentialItem( credential_name=db_credential.credential_name, - credential_values=decrypted_values, # pyright: ignore[reportArgumentType] # declared dict[str, str], and pydantic copies this mapping into one on validation; LIT002 rules out building that dict here + credential_values=decrypted_values(db_credential.credential_values), # pyright: ignore[reportArgumentType] # declared dict[str, str], and pydantic copies this mapping into one on validation; LIT002 rules out building that dict here credential_info=db_credential.credential_info, ) diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index 1472e380010..cb2be0e672d 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -263,6 +263,12 @@ def needs_legacy_reader(value: object) -> bool: return isinstance(value, str) and value != "" and not is_versioned_gcm(value) +def legacy_unreadable(value: object) -> bool: + """True for a stored value that only PyNaCl could read and PyNaCl is missing: such a value must read as unset, + never as a plaintext credential.""" + return needs_legacy_reader(value) and not legacy_encryption_available() + + def require_legacy_reader_for(values: Iterable[object], purpose: str) -> None: """Refuse a decrypt-then-rewrite pass when PyNaCl is missing and one of the values is unprefixed: it would read as unreadable and be dropped, double wrapped or miscounted as plaintext. Versioned gcm and non string diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index dc7f5699982..05d0de23a13 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -495,7 +495,7 @@ def _mcp_encrypted_leaves(col: str, raw: object) -> Iterator[str]: """Only the strings an MCP column encrypts at rest: a credentials blob keeps auth_type, scopes and urls in plaintext and env_vars keeps every name and every per user placeholder, so without PyNaCl those must not be mistaken for legacy ciphertext and refuse the scan""" - from litellm.proxy._experimental.mcp_server.db import MCP_CREDENTIAL_SECRET_FIELDS, _is_global_env_var_scope + from litellm.proxy._experimental.mcp_server.db import MCP_CREDENTIAL_SECRET_FIELDS, is_global_env_var_scope if col == "credentials": if not isinstance(raw, dict): @@ -506,7 +506,7 @@ def _mcp_encrypted_leaves(col: str, raw: object) -> Iterator[str]: return ( e["value"] for e in raw - if isinstance(e, dict) and _is_global_env_var_scope(e.get("scope")) and isinstance(e.get("value"), str) + if isinstance(e, dict) and is_global_env_var_scope(e.get("scope")) and isinstance(e.get("value"), str) ) @@ -609,7 +609,7 @@ async def _scan_config_env_vars(prisma_client: object) -> LocationReport: return report -async def _scan_covered_tables(prisma_client: object) -> list[LocationReport]: +async def scan_covered_tables(prisma_client: object) -> list[LocationReport]: """Read-only classification of every rotation-covered table. No writes.""" reports: Final[list[LocationReport]] = [] for location, db_attr, json_cols, scalar_cols in _COVERED_TABLE_SPECS: @@ -642,7 +642,7 @@ async def _migrate_covered_tables(prisma_client: object, user_api_key_dict: obje _rotate_master_key, ) - pre: Final = {r.location: r for r in await _scan_covered_tables(prisma_client)} + pre: Final = {r.location: r for r in await scan_covered_tables(prisma_client)} current_key: Final = _get_salt_key() if current_key is None: @@ -656,7 +656,7 @@ async def _migrate_covered_tables(prisma_client: object, user_api_key_dict: obje new_master_key=current_key, # same key, algorithm-only switch ) - post: Final = await _scan_covered_tables(prisma_client) + post: Final = await scan_covered_tables(prisma_client) for post_report in post: pre_report = pre.get(post_report.location) pre_legacy = pre_report.legacy if pre_report else 0 @@ -689,7 +689,7 @@ async def migrate_encryption( # delegate to the rotation path (with bracketing scans for counts); on a dry # run only classify them read-only. if dry_run: - for covered in await _scan_covered_tables(prisma_client): + for covered in await scan_covered_tables(prisma_client): report.add(covered) else: for covered in await _migrate_covered_tables(prisma_client, user_api_key_dict): @@ -717,7 +717,7 @@ async def check_encryption(prisma_client: object) -> MigrationReport: report: Final = MigrationReport() # Rotation-covered tables (read-only classification). - for covered in await _scan_covered_tables(prisma_client): + for covered in await scan_covered_tables(prisma_client): report.add(covered) # Net-new walker locations, in dry-run (read-only) mode. diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 5adb0f96d0d..531c8a6969c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -5222,9 +5222,9 @@ async def _require_legacy_reader_for_stored_values(prisma_client: PrismaClient) rotation before any row is rewritten. A fully migrated store passes; with PyNaCl installed nothing runs""" if legacy_encryption_available(): return - from litellm.proxy.management_endpoints.credential_migration import _scan_covered_tables + from litellm.proxy.management_endpoints.credential_migration import scan_covered_tables - await _scan_covered_tables(prisma_client) + await scan_covered_tables(prisma_client) async def _rotate_master_key( @@ -5255,7 +5255,7 @@ async def _rotate_master_key( except LegacyEncryptionUnavailableError as error: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={"error": str(error)}, # mutable-ok: FastAPI detail contract + detail={"error": str(error)}, ) from error try: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index df6f478ec45..fef11f521a4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -413,7 +413,7 @@ from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_pr from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber -from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored +from litellm.proxy.common_utils.credential_hydration import decrypted_values from litellm.proxy.common_utils.debug_utils import init_verbose_loggers from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.discoverable_model_filter import discoverable_rows, undiscoverable_model_names @@ -9155,11 +9155,7 @@ class ProxyConfig: elif isinstance(credential, BaseModel): credential_object = CredentialItem(**credential.model_dump()) - decrypted_credential_values: Final = {} - for k, v in credential_object.credential_values.items(): - decrypted_credential_values[k] = decrypted_or_stored(k, v) - - credential_object.credential_values = decrypted_credential_values + credential_object.credential_values = dict(decrypted_values(credential_object.credential_values)) return credential_object async def delete_credentials(self, db_credentials: list[CredentialItem]): diff --git a/tests/unit/proxy/common_utils/test_credential_hydration.py b/tests/unit/proxy/common_utils/test_credential_hydration.py index f40b0114d71..5a811c0cd6c 100644 --- a/tests/unit/proxy/common_utils/test_credential_hydration.py +++ b/tests/unit/proxy/common_utils/test_credential_hydration.py @@ -1,3 +1,6 @@ +import base64 +import hashlib +import sys from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -26,3 +29,32 @@ async def test_authoritative_hydrate_returns_an_encrypted_empty_value_as_empty(m assert resolved is not None assert resolved.credential_values == {"api_base": "", "openai_service_account_id": "user-1"} + + +def _legacy_nacl_ciphertext(plaintext: str, salt_key: str) -> str: + import nacl.secret + + box = nacl.secret.SecretBox(hashlib.sha256(salt_key.encode()).digest()) + return base64.urlsafe_b64encode(bytes(box.encrypt(plaintext.encode()))).decode() + + +@pytest.mark.asyncio +async def test_authoritative_hydrate_without_pynacl_drops_a_legacy_value_instead_of_serving_the_blob(monkeypatch): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-hydration-test-salt") + legacy = _legacy_nacl_ciphertext("sk-legacy-upstream", "sk-hydration-test-salt") + row = { + "credential_name": "openai-legacy", + "credential_values": {"api_key": legacy, "api_base": encrypt_value_helper("https://api.example.test")}, + "credential_info": {"custom_llm_provider": "openai"}, + } + prisma = MagicMock() + prisma.db.litellm_credentialstable.find_unique = AsyncMock(return_value=row) + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + with patch.object(litellm, "credential_list", []): # test-quality-ok: the row under test must win over memory + resolved = await hydrate_named_credential_authoritative("openai-legacy", prisma) + + assert resolved is not None + assert resolved.credential_values == {"api_base": "https://api.example.test"} + assert legacy not in resolved.credential_values.values() diff --git a/tests/unit/proxy/management_endpoints/test_credential_migration.py b/tests/unit/proxy/management_endpoints/test_credential_migration.py index 2e9d09c4ea9..070556d83b2 100644 --- a/tests/unit/proxy/management_endpoints/test_credential_migration.py +++ b/tests/unit/proxy/management_endpoints/test_credential_migration.py @@ -562,7 +562,7 @@ async def test_scan_covered_tables_classifies_legacy_and_v2(salt_key, monkeypatc ) client.db.litellm_config.find_unique = AsyncMock(return_value=None) - by_loc = {r.location: r for r in await cm._scan_covered_tables(client)} + by_loc = {r.location: r for r in await cm.scan_covered_tables(client)} assert by_loc["model_table"].legacy == 1 assert by_loc["model_table"].plaintext == 1 # "gpt-4" model name, not ciphertext @@ -586,7 +586,7 @@ async def test_scan_covered_tables_classifies_search_tool_params(salt_key, monke ) client.db.litellm_config.find_unique = AsyncMock(return_value=None) - by_loc = {r.location: r for r in await cm._scan_covered_tables(client)} + by_loc = {r.location: r for r in await cm.scan_covered_tables(client)} assert (by_loc["search_tools"].legacy, by_loc["search_tools"].already_v2) == (1, 1) assert by_loc["search_tools"].plaintext == 1 diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 6986ff0fb8d..d6e3fa8b065 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -3348,6 +3348,28 @@ def test_ProxyConfig_decrypt_credentials_returns_an_encrypted_empty_value_as_emp assert decrypted.credential_values == {"api_base": "", "openai_service_account_id": "user-1"} +def test_ProxyConfig_decrypt_credentials_without_pynacl_drops_a_legacy_value_instead_of_serving_the_blob(monkeypatch): + import base64 + import hashlib + + import nacl.secret + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-decrypt-credentials-test-salt") + box = nacl.secret.SecretBox(hashlib.sha256(b"sk-decrypt-credentials-test-salt").digest()) + legacy = base64.urlsafe_b64encode(bytes(box.encrypt(b"sk-legacy-upstream"))).decode() + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + decrypted = ProxyConfig().decrypt_credentials( + { + "credential_name": "openai-legacy", + "credential_values": {"api_key": legacy, "api_base": encrypt_value_helper("https://api.example.test")}, + "credential_info": {"custom_llm_provider": "openai"}, + } + ) + assert decrypted.credential_values == {"api_base": "https://api.example.test"} + + def test_ProxyConfig_decrypt_model_list_from_db_returns_decrypted(monkeypatch): monkeypatch.setattr( "litellm.proxy.proxy_server.decrypt_value_helper", From 0c2553cf6db8a6c2863663bd92fbc2984aec41d5 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 5 Oct 2026 08:54:36 +0000 Subject: [PATCH 17/17] refactor(proxy): build the credential object in one expression so CodeQL sees it always initialized Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index fef11f521a4..a19fdc3d3d4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9150,11 +9150,9 @@ class ProxyConfig: await initialize_pass_through_endpoints_in_db() def decrypt_credentials(self, credential: dict | BaseModel) -> CredentialItem: - if isinstance(credential, dict): - credential_object = CredentialItem(**credential) - elif isinstance(credential, BaseModel): - credential_object = CredentialItem(**credential.model_dump()) - + credential_object: Final = CredentialItem( + **(credential if isinstance(credential, dict) else credential.model_dump()) + ) credential_object.credential_values = dict(decrypted_values(credential_object.credential_values)) return credential_object