From 4836dc11c04c974fdbfc5f00d19aa5dbce65c9f2 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:01:43 -0700 Subject: [PATCH] =?UTF-8?q?fix(proxy):=20address=20review=20findings=20?= =?UTF-8?q?=E2=80=94=20nested=20guardrail=20scan=20leaves,=20refused-opt-i?= =?UTF-8?q?n=20plaintext=20fallback,=20boot-migration=20leaf=20selection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings from the bots on the previous head: 1. Guardrail scan skipped list-nested secrets (Bugbot): the guardrails writer encrypts sensitive-keyed strings inside arrays under the parent key, but _iter_keyed_strings tagged list items keyless so those leaves were never classified — a store could show residual_legacy == 0 while wrong-key or nacl leaves remained. List items now inherit the enclosing dict key. 2. Secret writers fell back to plaintext when the xsalsa20-poly1305 opt-in could not be honored (litereview, High): _encrypted_param, _encrypted_search_tool_value and _encrypt_if_plaintext wrapped encrypt_value_helper in a blanket except meant for the missing-salt-key case, so under FIPS mode or without PyNaCl the new FipsModeError / LegacyEncryptionUnavailableError were swallowed and the secret was persisted unencrypted. Both errors now re-raise; only the salt-key fallback keeps storing the value as written. 3. The LITELLM_MIGRATE_FROM_MASTER_KEY boot migration never finished on a no-PyNaCl install (litereview, Medium): counting walked every string leaf of every secret column, so plaintext noise (config settings rows, model tags, guardrail modes) read as values encrypted with the previous key and remaining never reached 0 — with an install-the-extra remedy a FIPS image cannot act on. The count now selects the leaves each column's writers actually encrypt: only the environment_variables config row, top-level litellm_params strings for the model/search-tool tables, and marker-prefixed sensitive guardrail leaves; other columns keep the conservative every-leaf walk. --- litellm/proxy/common_utils/callback_utils.py | 6 + litellm/proxy/db/master_key_migration.py | 105 +++++++++++++++--- .../proxy/guardrails/guardrail_registry.py | 11 +- .../credential_migration.py | 12 +- .../search_endpoints/search_tool_registry.py | 6 + .../test_encrypt_decrypt_utils.py | 66 +++++++++++ .../proxy/db/test_master_key_migration.py | 84 ++++++++++++++ .../test_credential_migration.py | 4 + 8 files changed, 273 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 4fa7709e5c7..4d8bc2a79b2 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -27,9 +27,11 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.proxy._types import CommonProxyErrors, LiteLLMPromptInjectionParams from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + LegacyEncryptionUnavailableError, decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.types_utils.utils import get_instance_fn from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( @@ -775,6 +777,10 @@ def _encrypt_if_plaintext(key: str, value: object) -> object: return value try: return _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value) + except (FipsModeError, LegacyEncryptionUnavailableError): + # The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or + # without PyNaCl): refuse the write instead of persisting the secret unencrypted. + raise except Exception: # No salt key / master key configured — leave the value as-is rather # than crash the write. Dev environments without LITELLM_SALT_KEY hit diff --git a/litellm/proxy/db/master_key_migration.py b/litellm/proxy/db/master_key_migration.py index 05e5a2d8166..441b05792d3 100644 --- a/litellm/proxy/db/master_key_migration.py +++ b/litellm/proxy/db/master_key_migration.py @@ -1,8 +1,8 @@ import json -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping from dataclasses import dataclass from enum import Enum -from typing import Final +from typing import Final, cast from pydantic import JsonValue, TypeAdapter from typing_extensions import assert_never @@ -86,13 +86,7 @@ async def count_values_encrypted_with(database: SupportsRawQueries, signing_key: # LITELLM_MIGRATE_FROM_MASTER_KEY while values under that key remain, unreadable. return value if legacy_unreadable(value) else None - return sum( - [ - replace_ciphertexts(_STORED_VALUE.validate_python(row[secret_column.column]), keep)[1] - for secret_column in await _secret_columns_in(database) - for row in await _rows_of(database, secret_column) - ] - ) + return await _count_secret_leaves(database, lambda leaf: keep(leaf) is not None) async def count_values_encrypted_with_or_none( @@ -111,17 +105,96 @@ async def _count_values_legacy_unreadable(database: SupportsRawQueries) -> int: encrypted-with-previous-key count stays conservative and this walk explains the residual.""" if legacy_encryption_available(): return 0 + return await _count_secret_leaves(database, legacy_unreadable) - def unreadable(value: str) -> str | None: - return value if legacy_unreadable(value) else None - async def unreadable_in(secret_column: _SecretColumn) -> int: - return sum( - replace_ciphertexts(_STORED_VALUE.validate_python(row[secret_column.column]), unreadable)[1] +async def _count_secret_leaves(database: SupportsRawQueries, counted: Callable[[str], bool]) -> int: + """Count the writer-encrypted secret leaves across every secret column that satisfy ``counted``.""" + + def counted_in(secret_column: _SecretColumn, row: Mapping[str, object]) -> int: + stored: Final = _STORED_VALUE.validate_python(row[secret_column.column]) + leaves: Final = _counted_secret_leaves(secret_column, row[secret_column.primary_key], stored) + return sum(1 for leaf in leaves if counted(leaf)) + + return sum( + [ + counted_in(secret_column, row) + for secret_column in await _secret_columns_in(database) for row in await _rows_of(database, secret_column) - ) + ] + ) - return sum([await unreadable_in(secret_column) for secret_column in await _secret_columns_in(database)]) + +# Columns whose writers encrypt each top-level string param and leave everything nested +# inside a list/dict param (tags, routing metadata, …) stored as plaintext. +_LITELLM_PARAMS_SECRET_COLUMNS: Final = frozenset( + ( + ("LiteLLM_ProxyModelTable", "litellm_params"), + ("LiteLLM_SearchToolsTable", "litellm_params"), + ) +) +_ENV_VARS_CONFIG_ROW: Final = "environment_variables" + + +def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, stored: JsonValue) -> Iterator[str]: + """Yield the string leaves of one stored value the writers of this column actually encrypt. + + Telling ciphertext from plaintext needs PyNaCl for unprefixed values, so a count over + every string leaf counts plaintext noise (config booleans, model tags, guardrail modes) + as possibly-encrypted-with-the-previous-key and the boot migration then never finishes + on an install that cannot install PyNaCl (a FIPS image). Restricting the candidates to + the leaves each column's writers encrypt keeps that noise out — the same per-column + selection the credential-migration scanner applies. Columns without a known writer + shape keep the conservative every-string-leaf walk. The callback-encryption marker is + stripped exactly as ``replace_ciphertexts`` strips it before the decrypt attempt.""" + if secret_column.table == "LiteLLM_Config" and _PRIMARY_KEY.validate_python(primary_key) != _ENV_VARS_CONFIG_ROW: + # Only the environment_variables row's values are ever encrypted; every other + # config row (general_settings, …) stores plaintext by construction. + return + if (secret_column.table, secret_column.column) in _LITELLM_PARAMS_SECRET_COLUMNS: + params: Final[JsonValue] = stored + if isinstance(params, dict): + for value in params.values(): + if isinstance(value, str): + yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) + return + if secret_column.table == "LiteLLM_GuardrailsTable": + yield from _sensitive_prefixed_leaves(stored) + return + yield from _string_leaves(stored) + + +def _string_leaves(value: JsonValue) -> Iterator[str]: + """Every string leaf of a nested structure, marker-stripped, via an explicit stack.""" + stack: Final[list[JsonValue]] = [value] # mutable-ok: walker stack + while stack: + cur = stack.pop() + if isinstance(cur, str): + yield cur.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) + elif isinstance(cur, (list, dict)): + stack.extend(cur.values() if isinstance(cur, dict) else cur) + + +def _sensitive_prefixed_leaves(value: JsonValue) -> Iterator[str]: + """Sensitive-keyed, marker-prefixed string leaves at any depth of a guardrails row. + + The guardrails writer stores every sensitive-keyed string with the callback + encryption marker — including strings inside lists under the sensitive key — and + everything else as plaintext by construction, so only marked leaves are candidates.""" + from litellm.proxy.common_utils.callback_utils import is_sensitive_callback_key + + stack: Final[list[tuple[str | None, JsonValue]]] = [(None, value)] # mutable-ok: walker stack + while stack: + key, cur = stack.pop() + if isinstance(cur, str): + if key is not None and is_sensitive_callback_key(key) and cur.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX): + yield cur.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) + elif isinstance(cur, dict): + entries = cast("dict[str, JsonValue]", cur) # cast-ok: JSON object keys are strings + stack.extend(entries.items()) + elif isinstance(cur, list): + items = cast("list[JsonValue]", cur) # cast-ok: JSON arrays hold JSON values + stack.extend((key, item) for item in items) async def reencrypt_stored_values(database: SupportsRawQueries, *, from_key: str, to_key: str) -> int: diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 7521c1e9f45..903dc1a952f 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -24,7 +24,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( ) from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX, is_sensitive_callback_key -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + LegacyEncryptionUnavailableError, + decrypt_value_helper, + encrypt_value_helper, +) +from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrail, ) @@ -133,6 +138,10 @@ def _encrypted_param(key: str, value: object, new_encryption_key: str | None, de return value try: return CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value, new_encryption_key=new_encryption_key) + except (FipsModeError, LegacyEncryptionUnavailableError): + # The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or + # without PyNaCl): refuse the write instead of persisting the secret unencrypted. + raise except Exception: # noqa: BLE001 # no salt key or master key configured: store the value as written return value diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index a1ebfefb3df..6636cc6f55b 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -617,10 +617,14 @@ def _top_level_param_leaves(params: Mapping[str, object]) -> Iterator[tuple[str, def _iter_keyed_strings(obj: object) -> Iterator[tuple[str | None, str]]: - """Yield every (dict key, string leaf) pair in a nested structure, keyless for list items. + """Yield every (dict key, string leaf) pair in a nested structure. - Iterative like ``_iter_encrypted_strings``: the code-quality recursive-function - detector bans recursion here, and an explicit stack walks arbitrary depth safely. + A list item inherits the key of the dict entry that holds the list: the guardrails + writer encrypts sensitive-keyed strings inside arrays under the parent key, so + ``["litellm_enc::"]`` under ``api_key`` is a keyed secret leaf, while items of a + root-level list stay keyless. Iterative like ``_iter_encrypted_strings``: the + code-quality recursive-function detector bans recursion here, and an explicit stack + walks arbitrary depth safely. """ stack: Final[list[tuple[str | None, object]]] = [(None, obj)] # mutable-ok: walker stack while stack: @@ -632,7 +636,7 @@ def _iter_keyed_strings(obj: object) -> Iterator[tuple[str | None, str]]: stack.extend(entries.items()) elif isinstance(cur, list): items = cast("list[object]", cur) # cast-ok: list items are walked as opaque objects - stack.extend((None, item) for item in items) + stack.extend((key, item) for item in items) def _guardrail_encrypted_leaves(raw: object) -> Iterator[tuple[str, bool]]: diff --git a/litellm/proxy/search_endpoints/search_tool_registry.py b/litellm/proxy/search_endpoints/search_tool_registry.py index 7b2e0f98bff..7ff3fa9bcc3 100644 --- a/litellm/proxy/search_endpoints/search_tool_registry.py +++ b/litellm/proxy/search_endpoints/search_tool_registry.py @@ -13,12 +13,14 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # legacy module exports + LegacyEncryptionUnavailableError, _get_salt_key, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export decrypt_if_encrypted_with, encrypt_value_helper, get_salt_key, legacy_unreadable, ) +from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry from litellm.proxy.utils import PrismaClient from litellm.repositories.table_repositories import SearchToolsRepository @@ -77,6 +79,10 @@ def _encrypted_search_tool_value(value: object) -> object: return value try: return encrypt_value_helper(value=value) + except (FipsModeError, LegacyEncryptionUnavailableError): + # The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or + # without PyNaCl): refuse the write instead of persisting the secret unencrypted. + raise except Exception: # noqa: BLE001 # no salt key or master key configured: store the value as written return value diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index 5f0a7a27bc9..4e336bf01b4 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -391,3 +391,69 @@ def test_bearer_token_uses_only_header_safe_characters(length: int): assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length + + +# --------------------------------------------------------------------------- +# Secret writers must refuse (not silently store plaintext) when the +# xsalsa20-poly1305 opt-in cannot be honored: PyNaCl missing or FIPS mode. +# --------------------------------------------------------------------------- + + +def _opt_into_xsalsa20(monkeypatch) -> None: + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) + + +def _without_pynacl(monkeypatch) -> None: + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + +def test_guardrail_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch): + """encrypt_guardrail_litellm_params used to swallow LegacyEncryptionUnavailableError + and persist the secret as plaintext; it must refuse the write instead.""" + _opt_into_xsalsa20(monkeypatch) + _without_pynacl(monkeypatch) + from litellm.proxy.guardrails.guardrail_registry import encrypt_guardrail_litellm_params + + with pytest.raises(LegacyEncryptionUnavailableError): + encrypt_guardrail_litellm_params({"api_key": "sk-plain-secret"}) + + +def test_search_tool_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch): + _opt_into_xsalsa20(monkeypatch) + _without_pynacl(monkeypatch) + from litellm.proxy.search_endpoints.search_tool_registry import _encrypted_search_tool_value + + with pytest.raises(LegacyEncryptionUnavailableError): + _encrypted_search_tool_value("sk-plain-secret") + + +def test_callback_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch): + _opt_into_xsalsa20(monkeypatch) + _without_pynacl(monkeypatch) + from litellm.proxy.common_utils.callback_utils import _encrypt_if_plaintext + + with pytest.raises(LegacyEncryptionUnavailableError): + _encrypt_if_plaintext("langfuse_public_key", "sk-plain-secret") + + +def test_guardrail_writer_refuses_xsalsa20_optin_under_fips(monkeypatch): + _opt_into_xsalsa20(monkeypatch) + monkeypatch.setenv("LITELLM_FIPS_MODE", "1") + from litellm.proxy.guardrails.guardrail_registry import encrypt_guardrail_litellm_params + + with pytest.raises(FipsModeError): + encrypt_guardrail_litellm_params({"api_key": "sk-plain-secret"}) + + +def test_secret_writers_still_encrypt_under_default_algorithm_without_pynacl(monkeypatch): + """Control: AES (the default) needs no PyNaCl, so a no-PyNaCl install still + encrypts secrets on every writer — only the refused opt-in changes behavior.""" + _use_aes(monkeypatch) + _without_pynacl(monkeypatch) + from litellm.proxy.common_utils.callback_utils import _encrypt_if_plaintext + from litellm.proxy.search_endpoints.search_tool_registry import _encrypted_search_tool_value + + assert _encrypted_search_tool_value("sk-plain-secret").startswith("v3:gcm:") + encrypted = _encrypt_if_plaintext("langfuse_public_key", "sk-plain-secret") + assert isinstance(encrypted, str) and encrypted.startswith("litellm_enc::v3:gcm:") diff --git a/tests/unit/proxy/db/test_master_key_migration.py b/tests/unit/proxy/db/test_master_key_migration.py index 4b7f61eaed1..5fce9dbfac1 100644 --- a/tests/unit/proxy/db/test_master_key_migration.py +++ b/tests/unit/proxy/db/test_master_key_migration.py @@ -648,3 +648,87 @@ async def test_guardrail_params_move_to_the_new_key_and_legacy_plaintext_rows_ar assert decrypt_if_encrypted_with(migrated_key.removeprefix("litellm_enc::"), NEW_KEY) == "guardrail-vendor-key" assert tables["LiteLLM_GuardrailsTable"][1]["litellm_params"] == legacy_params assert database.writes == [("LiteLLM_GuardrailsTable", "litellm_params", "guardrail-1")] + + +@pytest.mark.asyncio +async def test_plaintext_config_and_nested_model_leaves_do_not_block_a_pynacl_free_migration(monkeypatch): + """Without PyNaCl the count may only consider leaves the column's writers actually + encrypt. Plaintext noise — a config settings row, model tags — must not read as + values encrypted with the previous key, or the boot migration never finishes and a + FIPS image (which cannot install PyNaCl) is told forever to install the extra.""" + import sys + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + tables: Tables = { + "LiteLLM_Config": [ + {"param_name": "general_settings", "param_value": {"store_model_in_db": "true", "alerting": ["slack"]}}, + ], + "LiteLLM_ProxyModelTable": [ + { + "model_id": "model-1", + "litellm_params": { + "api_key": _encrypted("provider-key"), + "tags": ["prod"], + }, + } + ], + } + database = _FakeDatabase(tables) + + assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # only the api_key + + outcome = await migrate_from_previous_master_key( + previous_master_key=PREVIOUS_KEY, + master_key=NEW_KEY, + salt_key_is_set=False, + database=database, + log=lambda message: None, + ) + + # The one readable value migrated; the recount sees no previous-key value anywhere, + # so the operator is told the variable can go instead of being stuck on the extra. + assert outcome == Migrated(migrated=1, remaining=0, unreadable=0) + assert "You may now delete" in describe_outcome(outcome) + + second_boot = await migrate_from_previous_master_key( + previous_master_key=PREVIOUS_KEY, + master_key=NEW_KEY, + salt_key_is_set=False, + database=database, + log=lambda message: None, + ) + assert second_boot == NothingToMigrate.NOTHING_ENCRYPTED_WITH_PREVIOUS_KEY + assert ("LiteLLM_ProxyModelTable", "litellm_params", "model-1") in database.writes + row = tables["LiteLLM_ProxyModelTable"][0]["litellm_params"] + assert row["tags"] == ["prod"] # plaintext nested leaf untouched + assert decrypt_if_encrypted_with(str(row["api_key"]), NEW_KEY) == "provider-key" + assert tables["LiteLLM_Config"][0]["param_value"] == { + "store_model_in_db": "true", + "alerting": ["slack"], + } # plaintext config row untouched + + +@pytest.mark.asyncio +async def test_guardrail_mode_and_unprefixed_params_do_not_block_a_pynacl_free_migration(monkeypatch): + """A guardrails row's plaintext params never read as previous-key ciphertext either: + its writer marks every encrypted value, so only marked sensitive leaves count.""" + import sys + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + tables: Tables = { + "LiteLLM_GuardrailsTable": [ + { + "guardrail_id": "gr-1", + "litellm_params": { + "mode": "pre_call", + "guardrail_name": "presidio", + "api_key": "litellm_enc::" + _encrypted("guardrail-key"), + }, + } + ], + } + database = _FakeDatabase(tables) + + assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # the marked api_key only diff --git a/tests/unit/proxy/management_endpoints/test_credential_migration.py b/tests/unit/proxy/management_endpoints/test_credential_migration.py index 966e6fb0db9..8c5a4a860d9 100644 --- a/tests/unit/proxy/management_endpoints/test_credential_migration.py +++ b/tests/unit/proxy/management_endpoints/test_credential_migration.py @@ -1126,6 +1126,10 @@ def _covered_table_rig( litellm_params={ "mode": "pre_call", "api_key": CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("guardrail-secret"), + # Nested shapes the guardrails writer also encrypts: a sensitive key + # inside a nested dict, and sensitive-keyed strings inside a list. + "metadata": {"api_key": CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("nested")}, + "secret_headers": [CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("listed")], }, ) )