fix(proxy): address review findings — nested guardrail scan leaves, refused-opt-in plaintext fallback, boot-migration leaf selection

Three review findings from the bots on the previous head:

1. Guardrail scan skipped list-nested secrets (Bugbot): the guardrails
   writer encrypts sensitive-keyed strings inside arrays under the parent
   key, but _iter_keyed_strings tagged list items keyless so those leaves
   were never classified — a store could show residual_legacy == 0 while
   wrong-key or nacl leaves remained. List items now inherit the
   enclosing dict key.

2. Secret writers fell back to plaintext when the xsalsa20-poly1305
   opt-in could not be honored (litereview, High): _encrypted_param,
   _encrypted_search_tool_value and _encrypt_if_plaintext wrapped
   encrypt_value_helper in a blanket except meant for the missing-salt-key
   case, so under FIPS mode or without PyNaCl the new FipsModeError /
   LegacyEncryptionUnavailableError were swallowed and the secret was
   persisted unencrypted. Both errors now re-raise; only the salt-key
   fallback keeps storing the value as written.

3. The LITELLM_MIGRATE_FROM_MASTER_KEY boot migration never finished on a
   no-PyNaCl install (litereview, Medium): counting walked every string
   leaf of every secret column, so plaintext noise (config settings rows,
   model tags, guardrail modes) read as values encrypted with the previous
   key and remaining never reached 0 — with an install-the-extra remedy a
   FIPS image cannot act on. The count now selects the leaves each
   column's writers actually encrypt: only the environment_variables
   config row, top-level litellm_params strings for the model/search-tool
   tables, and marker-prefixed sensitive guardrail leaves; other columns
   keep the conservative every-leaf walk.
This commit is contained in:
yucheng-berri 2026-10-14 21:01:43 -07:00
parent a5dd70a06c
commit 4836dc11c0
8 changed files with 273 additions and 21 deletions

View file

@ -27,9 +27,11 @@ from litellm.litellm_core_utils.core_helpers import (
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.proxy._types import CommonProxyErrors, LiteLLMPromptInjectionParams from litellm.proxy._types import CommonProxyErrors, LiteLLMPromptInjectionParams
from litellm.proxy.common_utils.encrypt_decrypt_utils import ( from litellm.proxy.common_utils.encrypt_decrypt_utils import (
LegacyEncryptionUnavailableError,
decrypt_value_helper, decrypt_value_helper,
encrypt_value_helper, encrypt_value_helper,
) )
from litellm.proxy.common_utils.fips import FipsModeError
from litellm.proxy.types_utils.utils import get_instance_fn from litellm.proxy.types_utils.utils import get_instance_fn
from litellm.types.guardrails import GuardrailEventHooks from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import ( from litellm.types.utils import (
@ -775,6 +777,10 @@ def _encrypt_if_plaintext(key: str, value: object) -> object:
return value return value
try: try:
return _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value) return _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value)
except (FipsModeError, LegacyEncryptionUnavailableError):
# The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or
# without PyNaCl): refuse the write instead of persisting the secret unencrypted.
raise
except Exception: except Exception:
# No salt key / master key configured — leave the value as-is rather # No salt key / master key configured — leave the value as-is rather
# than crash the write. Dev environments without LITELLM_SALT_KEY hit # than crash the write. Dev environments without LITELLM_SALT_KEY hit

View file

@ -1,8 +1,8 @@
import json import json
from collections.abc import Awaitable, Callable, Mapping from collections.abc import Awaitable, Callable, Iterator, Mapping
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum from enum import Enum
from typing import Final from typing import Final, cast
from pydantic import JsonValue, TypeAdapter from pydantic import JsonValue, TypeAdapter
from typing_extensions import assert_never from typing_extensions import assert_never
@ -86,13 +86,7 @@ async def count_values_encrypted_with(database: SupportsRawQueries, signing_key:
# LITELLM_MIGRATE_FROM_MASTER_KEY while values under that key remain, unreadable. # LITELLM_MIGRATE_FROM_MASTER_KEY while values under that key remain, unreadable.
return value if legacy_unreadable(value) else None return value if legacy_unreadable(value) else None
return sum( return await _count_secret_leaves(database, lambda leaf: keep(leaf) is not None)
[
replace_ciphertexts(_STORED_VALUE.validate_python(row[secret_column.column]), keep)[1]
for secret_column in await _secret_columns_in(database)
for row in await _rows_of(database, secret_column)
]
)
async def count_values_encrypted_with_or_none( async def count_values_encrypted_with_or_none(
@ -111,17 +105,96 @@ async def _count_values_legacy_unreadable(database: SupportsRawQueries) -> int:
encrypted-with-previous-key count stays conservative and this walk explains the residual.""" encrypted-with-previous-key count stays conservative and this walk explains the residual."""
if legacy_encryption_available(): if legacy_encryption_available():
return 0 return 0
return await _count_secret_leaves(database, legacy_unreadable)
def unreadable(value: str) -> str | None:
return value if legacy_unreadable(value) else None
async def unreadable_in(secret_column: _SecretColumn) -> int: async def _count_secret_leaves(database: SupportsRawQueries, counted: Callable[[str], bool]) -> int:
return sum( """Count the writer-encrypted secret leaves across every secret column that satisfy ``counted``."""
replace_ciphertexts(_STORED_VALUE.validate_python(row[secret_column.column]), unreadable)[1]
def counted_in(secret_column: _SecretColumn, row: Mapping[str, object]) -> int:
stored: Final = _STORED_VALUE.validate_python(row[secret_column.column])
leaves: Final = _counted_secret_leaves(secret_column, row[secret_column.primary_key], stored)
return sum(1 for leaf in leaves if counted(leaf))
return sum(
[
counted_in(secret_column, row)
for secret_column in await _secret_columns_in(database)
for row in await _rows_of(database, secret_column) for row in await _rows_of(database, secret_column)
) ]
)
return sum([await unreadable_in(secret_column) for secret_column in await _secret_columns_in(database)])
# Columns whose writers encrypt each top-level string param and leave everything nested
# inside a list/dict param (tags, routing metadata, …) stored as plaintext.
_LITELLM_PARAMS_SECRET_COLUMNS: Final = frozenset(
(
("LiteLLM_ProxyModelTable", "litellm_params"),
("LiteLLM_SearchToolsTable", "litellm_params"),
)
)
_ENV_VARS_CONFIG_ROW: Final = "environment_variables"
def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, stored: JsonValue) -> Iterator[str]:
"""Yield the string leaves of one stored value the writers of this column actually encrypt.
Telling ciphertext from plaintext needs PyNaCl for unprefixed values, so a count over
every string leaf counts plaintext noise (config booleans, model tags, guardrail modes)
as possibly-encrypted-with-the-previous-key and the boot migration then never finishes
on an install that cannot install PyNaCl (a FIPS image). Restricting the candidates to
the leaves each column's writers encrypt keeps that noise out — the same per-column
selection the credential-migration scanner applies. Columns without a known writer
shape keep the conservative every-string-leaf walk. The callback-encryption marker is
stripped exactly as ``replace_ciphertexts`` strips it before the decrypt attempt."""
if secret_column.table == "LiteLLM_Config" and _PRIMARY_KEY.validate_python(primary_key) != _ENV_VARS_CONFIG_ROW:
# Only the environment_variables row's values are ever encrypted; every other
# config row (general_settings, …) stores plaintext by construction.
return
if (secret_column.table, secret_column.column) in _LITELLM_PARAMS_SECRET_COLUMNS:
params: Final[JsonValue] = stored
if isinstance(params, dict):
for value in params.values():
if isinstance(value, str):
yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX)
return
if secret_column.table == "LiteLLM_GuardrailsTable":
yield from _sensitive_prefixed_leaves(stored)
return
yield from _string_leaves(stored)
def _string_leaves(value: JsonValue) -> Iterator[str]:
"""Every string leaf of a nested structure, marker-stripped, via an explicit stack."""
stack: Final[list[JsonValue]] = [value] # mutable-ok: walker stack
while stack:
cur = stack.pop()
if isinstance(cur, str):
yield cur.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX)
elif isinstance(cur, (list, dict)):
stack.extend(cur.values() if isinstance(cur, dict) else cur)
def _sensitive_prefixed_leaves(value: JsonValue) -> Iterator[str]:
"""Sensitive-keyed, marker-prefixed string leaves at any depth of a guardrails row.
The guardrails writer stores every sensitive-keyed string with the callback
encryption marker — including strings inside lists under the sensitive key — and
everything else as plaintext by construction, so only marked leaves are candidates."""
from litellm.proxy.common_utils.callback_utils import is_sensitive_callback_key
stack: Final[list[tuple[str | None, JsonValue]]] = [(None, value)] # mutable-ok: walker stack
while stack:
key, cur = stack.pop()
if isinstance(cur, str):
if key is not None and is_sensitive_callback_key(key) and cur.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX):
yield cur.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX)
elif isinstance(cur, dict):
entries = cast("dict[str, JsonValue]", cur) # cast-ok: JSON object keys are strings
stack.extend(entries.items())
elif isinstance(cur, list):
items = cast("list[JsonValue]", cur) # cast-ok: JSON arrays hold JSON values
stack.extend((key, item) for item in items)
async def reencrypt_stored_values(database: SupportsRawQueries, *, from_key: str, to_key: str) -> int: async def reencrypt_stored_values(database: SupportsRawQueries, *, from_key: str, to_key: str) -> int:

View file

@ -24,7 +24,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
) )
from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR
from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX, is_sensitive_callback_key from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX, is_sensitive_callback_key
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper from litellm.proxy.common_utils.encrypt_decrypt_utils import (
LegacyEncryptionUnavailableError,
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy.common_utils.fips import FipsModeError
from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
BedrockGuardrail, BedrockGuardrail,
) )
@ -133,6 +138,10 @@ def _encrypted_param(key: str, value: object, new_encryption_key: str | None, de
return value return value
try: try:
return CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value, new_encryption_key=new_encryption_key) return CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value, new_encryption_key=new_encryption_key)
except (FipsModeError, LegacyEncryptionUnavailableError):
# The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or
# without PyNaCl): refuse the write instead of persisting the secret unencrypted.
raise
except Exception: # noqa: BLE001 # no salt key or master key configured: store the value as written except Exception: # noqa: BLE001 # no salt key or master key configured: store the value as written
return value return value

View file

@ -617,10 +617,14 @@ def _top_level_param_leaves(params: Mapping[str, object]) -> Iterator[tuple[str,
def _iter_keyed_strings(obj: object) -> Iterator[tuple[str | None, str]]: def _iter_keyed_strings(obj: object) -> Iterator[tuple[str | None, str]]:
"""Yield every (dict key, string leaf) pair in a nested structure, keyless for list items. """Yield every (dict key, string leaf) pair in a nested structure.
Iterative like ``_iter_encrypted_strings``: the code-quality recursive-function A list item inherits the key of the dict entry that holds the list: the guardrails
detector bans recursion here, and an explicit stack walks arbitrary depth safely. writer encrypts sensitive-keyed strings inside arrays under the parent key, so
``["litellm_enc::<ct>"]`` under ``api_key`` is a keyed secret leaf, while items of a
root-level list stay keyless. Iterative like ``_iter_encrypted_strings``: the
code-quality recursive-function detector bans recursion here, and an explicit stack
walks arbitrary depth safely.
""" """
stack: Final[list[tuple[str | None, object]]] = [(None, obj)] # mutable-ok: walker stack stack: Final[list[tuple[str | None, object]]] = [(None, obj)] # mutable-ok: walker stack
while stack: while stack:
@ -632,7 +636,7 @@ def _iter_keyed_strings(obj: object) -> Iterator[tuple[str | None, str]]:
stack.extend(entries.items()) stack.extend(entries.items())
elif isinstance(cur, list): elif isinstance(cur, list):
items = cast("list[object]", cur) # cast-ok: list items are walked as opaque objects items = cast("list[object]", cur) # cast-ok: list items are walked as opaque objects
stack.extend((None, item) for item in items) stack.extend((key, item) for item in items)
def _guardrail_encrypted_leaves(raw: object) -> Iterator[tuple[str, bool]]: def _guardrail_encrypted_leaves(raw: object) -> Iterator[tuple[str, bool]]:

View file

@ -13,12 +13,14 @@ from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR
from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # legacy module exports from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # legacy module exports
LegacyEncryptionUnavailableError,
_get_salt_key, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export _get_salt_key, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
decrypt_if_encrypted_with, decrypt_if_encrypted_with,
encrypt_value_helper, encrypt_value_helper,
get_salt_key, get_salt_key,
legacy_unreadable, legacy_unreadable,
) )
from litellm.proxy.common_utils.fips import FipsModeError
from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
from litellm.proxy.utils import PrismaClient from litellm.proxy.utils import PrismaClient
from litellm.repositories.table_repositories import SearchToolsRepository from litellm.repositories.table_repositories import SearchToolsRepository
@ -77,6 +79,10 @@ def _encrypted_search_tool_value(value: object) -> object:
return value return value
try: try:
return encrypt_value_helper(value=value) return encrypt_value_helper(value=value)
except (FipsModeError, LegacyEncryptionUnavailableError):
# The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or
# without PyNaCl): refuse the write instead of persisting the secret unencrypted.
raise
except Exception: # noqa: BLE001 # no salt key or master key configured: store the value as written except Exception: # noqa: BLE001 # no salt key or master key configured: store the value as written
return value return value

View file

@ -391,3 +391,69 @@ def test_bearer_token_uses_only_header_safe_characters(length: int):
assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token
assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length
# ---------------------------------------------------------------------------
# Secret writers must refuse (not silently store plaintext) when the
# xsalsa20-poly1305 opt-in cannot be honored: PyNaCl missing or FIPS mode.
# ---------------------------------------------------------------------------
def _opt_into_xsalsa20(monkeypatch) -> None:
monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"})
def _without_pynacl(monkeypatch) -> None:
monkeypatch.setitem(sys.modules, "nacl", None)
monkeypatch.setitem(sys.modules, "nacl.secret", None)
def test_guardrail_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch):
"""encrypt_guardrail_litellm_params used to swallow LegacyEncryptionUnavailableError
and persist the secret as plaintext; it must refuse the write instead."""
_opt_into_xsalsa20(monkeypatch)
_without_pynacl(monkeypatch)
from litellm.proxy.guardrails.guardrail_registry import encrypt_guardrail_litellm_params
with pytest.raises(LegacyEncryptionUnavailableError):
encrypt_guardrail_litellm_params({"api_key": "sk-plain-secret"})
def test_search_tool_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch):
_opt_into_xsalsa20(monkeypatch)
_without_pynacl(monkeypatch)
from litellm.proxy.search_endpoints.search_tool_registry import _encrypted_search_tool_value
with pytest.raises(LegacyEncryptionUnavailableError):
_encrypted_search_tool_value("sk-plain-secret")
def test_callback_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch):
_opt_into_xsalsa20(monkeypatch)
_without_pynacl(monkeypatch)
from litellm.proxy.common_utils.callback_utils import _encrypt_if_plaintext
with pytest.raises(LegacyEncryptionUnavailableError):
_encrypt_if_plaintext("langfuse_public_key", "sk-plain-secret")
def test_guardrail_writer_refuses_xsalsa20_optin_under_fips(monkeypatch):
_opt_into_xsalsa20(monkeypatch)
monkeypatch.setenv("LITELLM_FIPS_MODE", "1")
from litellm.proxy.guardrails.guardrail_registry import encrypt_guardrail_litellm_params
with pytest.raises(FipsModeError):
encrypt_guardrail_litellm_params({"api_key": "sk-plain-secret"})
def test_secret_writers_still_encrypt_under_default_algorithm_without_pynacl(monkeypatch):
"""Control: AES (the default) needs no PyNaCl, so a no-PyNaCl install still
encrypts secrets on every writer — only the refused opt-in changes behavior."""
_use_aes(monkeypatch)
_without_pynacl(monkeypatch)
from litellm.proxy.common_utils.callback_utils import _encrypt_if_plaintext
from litellm.proxy.search_endpoints.search_tool_registry import _encrypted_search_tool_value
assert _encrypted_search_tool_value("sk-plain-secret").startswith("v3:gcm:")
encrypted = _encrypt_if_plaintext("langfuse_public_key", "sk-plain-secret")
assert isinstance(encrypted, str) and encrypted.startswith("litellm_enc::v3:gcm:")

View file

@ -648,3 +648,87 @@ async def test_guardrail_params_move_to_the_new_key_and_legacy_plaintext_rows_ar
assert decrypt_if_encrypted_with(migrated_key.removeprefix("litellm_enc::"), NEW_KEY) == "guardrail-vendor-key" assert decrypt_if_encrypted_with(migrated_key.removeprefix("litellm_enc::"), NEW_KEY) == "guardrail-vendor-key"
assert tables["LiteLLM_GuardrailsTable"][1]["litellm_params"] == legacy_params assert tables["LiteLLM_GuardrailsTable"][1]["litellm_params"] == legacy_params
assert database.writes == [("LiteLLM_GuardrailsTable", "litellm_params", "guardrail-1")] assert database.writes == [("LiteLLM_GuardrailsTable", "litellm_params", "guardrail-1")]
@pytest.mark.asyncio
async def test_plaintext_config_and_nested_model_leaves_do_not_block_a_pynacl_free_migration(monkeypatch):
"""Without PyNaCl the count may only consider leaves the column's writers actually
encrypt. Plaintext noise — a config settings row, model tags — must not read as
values encrypted with the previous key, or the boot migration never finishes and a
FIPS image (which cannot install PyNaCl) is told forever to install the extra."""
import sys
monkeypatch.setitem(sys.modules, "nacl", None)
monkeypatch.setitem(sys.modules, "nacl.secret", None)
tables: Tables = {
"LiteLLM_Config": [
{"param_name": "general_settings", "param_value": {"store_model_in_db": "true", "alerting": ["slack"]}},
],
"LiteLLM_ProxyModelTable": [
{
"model_id": "model-1",
"litellm_params": {
"api_key": _encrypted("provider-key"),
"tags": ["prod"],
},
}
],
}
database = _FakeDatabase(tables)
assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # only the api_key
outcome = await migrate_from_previous_master_key(
previous_master_key=PREVIOUS_KEY,
master_key=NEW_KEY,
salt_key_is_set=False,
database=database,
log=lambda message: None,
)
# The one readable value migrated; the recount sees no previous-key value anywhere,
# so the operator is told the variable can go instead of being stuck on the extra.
assert outcome == Migrated(migrated=1, remaining=0, unreadable=0)
assert "You may now delete" in describe_outcome(outcome)
second_boot = await migrate_from_previous_master_key(
previous_master_key=PREVIOUS_KEY,
master_key=NEW_KEY,
salt_key_is_set=False,
database=database,
log=lambda message: None,
)
assert second_boot == NothingToMigrate.NOTHING_ENCRYPTED_WITH_PREVIOUS_KEY
assert ("LiteLLM_ProxyModelTable", "litellm_params", "model-1") in database.writes
row = tables["LiteLLM_ProxyModelTable"][0]["litellm_params"]
assert row["tags"] == ["prod"] # plaintext nested leaf untouched
assert decrypt_if_encrypted_with(str(row["api_key"]), NEW_KEY) == "provider-key"
assert tables["LiteLLM_Config"][0]["param_value"] == {
"store_model_in_db": "true",
"alerting": ["slack"],
} # plaintext config row untouched
@pytest.mark.asyncio
async def test_guardrail_mode_and_unprefixed_params_do_not_block_a_pynacl_free_migration(monkeypatch):
"""A guardrails row's plaintext params never read as previous-key ciphertext either:
its writer marks every encrypted value, so only marked sensitive leaves count."""
import sys
monkeypatch.setitem(sys.modules, "nacl", None)
monkeypatch.setitem(sys.modules, "nacl.secret", None)
tables: Tables = {
"LiteLLM_GuardrailsTable": [
{
"guardrail_id": "gr-1",
"litellm_params": {
"mode": "pre_call",
"guardrail_name": "presidio",
"api_key": "litellm_enc::" + _encrypted("guardrail-key"),
},
}
],
}
database = _FakeDatabase(tables)
assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # the marked api_key only

View file

@ -1126,6 +1126,10 @@ def _covered_table_rig(
litellm_params={ litellm_params={
"mode": "pre_call", "mode": "pre_call",
"api_key": CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("guardrail-secret"), "api_key": CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("guardrail-secret"),
# Nested shapes the guardrails writer also encrypts: a sensitive key
# inside a nested dict, and sensitive-keyed strings inside a list.
"metadata": {"api_key": CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("nested")},
"secret_headers": [CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("listed")],
}, },
) )
) )