diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 4fa7709e5c7..4d8bc2a79b2 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -27,9 +27,11 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.proxy._types import CommonProxyErrors, LiteLLMPromptInjectionParams from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + LegacyEncryptionUnavailableError, decrypt_value_helper, encrypt_value_helper, ) +from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.types_utils.utils import get_instance_fn from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( @@ -775,6 +777,10 @@ def _encrypt_if_plaintext(key: str, value: object) -> object: return value try: return _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value) + except (FipsModeError, LegacyEncryptionUnavailableError): + # The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or + # without PyNaCl): refuse the write instead of persisting the secret unencrypted. + raise except Exception: # No salt key / master key configured — leave the value as-is rather # than crash the write. Dev environments without LITELLM_SALT_KEY hit diff --git a/litellm/proxy/db/master_key_migration.py b/litellm/proxy/db/master_key_migration.py index 05e5a2d8166..441b05792d3 100644 --- a/litellm/proxy/db/master_key_migration.py +++ b/litellm/proxy/db/master_key_migration.py @@ -1,8 +1,8 @@ import json -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping from dataclasses import dataclass from enum import Enum -from typing import Final +from typing import Final, cast from pydantic import JsonValue, TypeAdapter from typing_extensions import assert_never @@ -86,13 +86,7 @@ async def count_values_encrypted_with(database: SupportsRawQueries, signing_key: # LITELLM_MIGRATE_FROM_MASTER_KEY while values under that key remain, unreadable. return value if legacy_unreadable(value) else None - return sum( - [ - replace_ciphertexts(_STORED_VALUE.validate_python(row[secret_column.column]), keep)[1] - for secret_column in await _secret_columns_in(database) - for row in await _rows_of(database, secret_column) - ] - ) + return await _count_secret_leaves(database, lambda leaf: keep(leaf) is not None) async def count_values_encrypted_with_or_none( @@ -111,17 +105,96 @@ async def _count_values_legacy_unreadable(database: SupportsRawQueries) -> int: encrypted-with-previous-key count stays conservative and this walk explains the residual.""" if legacy_encryption_available(): return 0 + return await _count_secret_leaves(database, legacy_unreadable) - def unreadable(value: str) -> str | None: - return value if legacy_unreadable(value) else None - async def unreadable_in(secret_column: _SecretColumn) -> int: - return sum( - replace_ciphertexts(_STORED_VALUE.validate_python(row[secret_column.column]), unreadable)[1] +async def _count_secret_leaves(database: SupportsRawQueries, counted: Callable[[str], bool]) -> int: + """Count the writer-encrypted secret leaves across every secret column that satisfy ``counted``.""" + + def counted_in(secret_column: _SecretColumn, row: Mapping[str, object]) -> int: + stored: Final = _STORED_VALUE.validate_python(row[secret_column.column]) + leaves: Final = _counted_secret_leaves(secret_column, row[secret_column.primary_key], stored) + return sum(1 for leaf in leaves if counted(leaf)) + + return sum( + [ + counted_in(secret_column, row) + for secret_column in await _secret_columns_in(database) for row in await _rows_of(database, secret_column) - ) + ] + ) - return sum([await unreadable_in(secret_column) for secret_column in await _secret_columns_in(database)]) + +# Columns whose writers encrypt each top-level string param and leave everything nested +# inside a list/dict param (tags, routing metadata, …) stored as plaintext. +_LITELLM_PARAMS_SECRET_COLUMNS: Final = frozenset( + ( + ("LiteLLM_ProxyModelTable", "litellm_params"), + ("LiteLLM_SearchToolsTable", "litellm_params"), + ) +) +_ENV_VARS_CONFIG_ROW: Final = "environment_variables" + + +def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, stored: JsonValue) -> Iterator[str]: + """Yield the string leaves of one stored value the writers of this column actually encrypt. + + Telling ciphertext from plaintext needs PyNaCl for unprefixed values, so a count over + every string leaf counts plaintext noise (config booleans, model tags, guardrail modes) + as possibly-encrypted-with-the-previous-key and the boot migration then never finishes + on an install that cannot install PyNaCl (a FIPS image). Restricting the candidates to + the leaves each column's writers encrypt keeps that noise out — the same per-column + selection the credential-migration scanner applies. Columns without a known writer + shape keep the conservative every-string-leaf walk. The callback-encryption marker is + stripped exactly as ``replace_ciphertexts`` strips it before the decrypt attempt.""" + if secret_column.table == "LiteLLM_Config" and _PRIMARY_KEY.validate_python(primary_key) != _ENV_VARS_CONFIG_ROW: + # Only the environment_variables row's values are ever encrypted; every other + # config row (general_settings, …) stores plaintext by construction. + return + if (secret_column.table, secret_column.column) in _LITELLM_PARAMS_SECRET_COLUMNS: + params: Final[JsonValue] = stored + if isinstance(params, dict): + for value in params.values(): + if isinstance(value, str): + yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) + return + if secret_column.table == "LiteLLM_GuardrailsTable": + yield from _sensitive_prefixed_leaves(stored) + return + yield from _string_leaves(stored) + + +def _string_leaves(value: JsonValue) -> Iterator[str]: + """Every string leaf of a nested structure, marker-stripped, via an explicit stack.""" + stack: Final[list[JsonValue]] = [value] # mutable-ok: walker stack + while stack: + cur = stack.pop() + if isinstance(cur, str): + yield cur.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) + elif isinstance(cur, (list, dict)): + stack.extend(cur.values() if isinstance(cur, dict) else cur) + + +def _sensitive_prefixed_leaves(value: JsonValue) -> Iterator[str]: + """Sensitive-keyed, marker-prefixed string leaves at any depth of a guardrails row. + + The guardrails writer stores every sensitive-keyed string with the callback + encryption marker — including strings inside lists under the sensitive key — and + everything else as plaintext by construction, so only marked leaves are candidates.""" + from litellm.proxy.common_utils.callback_utils import is_sensitive_callback_key + + stack: Final[list[tuple[str | None, JsonValue]]] = [(None, value)] # mutable-ok: walker stack + while stack: + key, cur = stack.pop() + if isinstance(cur, str): + if key is not None and is_sensitive_callback_key(key) and cur.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX): + yield cur.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) + elif isinstance(cur, dict): + entries = cast("dict[str, JsonValue]", cur) # cast-ok: JSON object keys are strings + stack.extend(entries.items()) + elif isinstance(cur, list): + items = cast("list[JsonValue]", cur) # cast-ok: JSON arrays hold JSON values + stack.extend((key, item) for item in items) async def reencrypt_stored_values(database: SupportsRawQueries, *, from_key: str, to_key: str) -> int: diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 7521c1e9f45..903dc1a952f 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -24,7 +24,12 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( ) from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX, is_sensitive_callback_key -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + LegacyEncryptionUnavailableError, + decrypt_value_helper, + encrypt_value_helper, +) +from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( BedrockGuardrail, ) @@ -133,6 +138,10 @@ def _encrypted_param(key: str, value: object, new_encryption_key: str | None, de return value try: return CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper(value, new_encryption_key=new_encryption_key) + except (FipsModeError, LegacyEncryptionUnavailableError): + # The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or + # without PyNaCl): refuse the write instead of persisting the secret unencrypted. + raise except Exception: # noqa: BLE001 # no salt key or master key configured: store the value as written return value diff --git a/litellm/proxy/management_endpoints/credential_migration.py b/litellm/proxy/management_endpoints/credential_migration.py index a1ebfefb3df..6636cc6f55b 100644 --- a/litellm/proxy/management_endpoints/credential_migration.py +++ b/litellm/proxy/management_endpoints/credential_migration.py @@ -617,10 +617,14 @@ def _top_level_param_leaves(params: Mapping[str, object]) -> Iterator[tuple[str, def _iter_keyed_strings(obj: object) -> Iterator[tuple[str | None, str]]: - """Yield every (dict key, string leaf) pair in a nested structure, keyless for list items. + """Yield every (dict key, string leaf) pair in a nested structure. - Iterative like ``_iter_encrypted_strings``: the code-quality recursive-function - detector bans recursion here, and an explicit stack walks arbitrary depth safely. + A list item inherits the key of the dict entry that holds the list: the guardrails + writer encrypts sensitive-keyed strings inside arrays under the parent key, so + ``["litellm_enc::"]`` under ``api_key`` is a keyed secret leaf, while items of a + root-level list stay keyless. Iterative like ``_iter_encrypted_strings``: the + code-quality recursive-function detector bans recursion here, and an explicit stack + walks arbitrary depth safely. """ stack: Final[list[tuple[str | None, object]]] = [(None, obj)] # mutable-ok: walker stack while stack: @@ -632,7 +636,7 @@ def _iter_keyed_strings(obj: object) -> Iterator[tuple[str | None, str]]: stack.extend(entries.items()) elif isinstance(cur, list): items = cast("list[object]", cur) # cast-ok: list items are walked as opaque objects - stack.extend((None, item) for item in items) + stack.extend((key, item) for item in items) def _guardrail_encrypted_leaves(raw: object) -> Iterator[tuple[str, bool]]: diff --git a/litellm/proxy/search_endpoints/search_tool_registry.py b/litellm/proxy/search_endpoints/search_tool_registry.py index 7b2e0f98bff..7ff3fa9bcc3 100644 --- a/litellm/proxy/search_endpoints/search_tool_registry.py +++ b/litellm/proxy/search_endpoints/search_tool_registry.py @@ -13,12 +13,14 @@ from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR from litellm.proxy.common_utils.encrypt_decrypt_utils import ( # noqa: F401 # legacy module exports + LegacyEncryptionUnavailableError, _get_salt_key, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export decrypt_if_encrypted_with, encrypt_value_helper, get_salt_key, legacy_unreadable, ) +from litellm.proxy.common_utils.fips import FipsModeError from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry from litellm.proxy.utils import PrismaClient from litellm.repositories.table_repositories import SearchToolsRepository @@ -77,6 +79,10 @@ def _encrypted_search_tool_value(value: object) -> object: return value try: return encrypt_value_helper(value=value) + except (FipsModeError, LegacyEncryptionUnavailableError): + # The configured algorithm cannot be honored (xsalsa20 opt-in under FIPS mode, or + # without PyNaCl): refuse the write instead of persisting the secret unencrypted. + raise except Exception: # noqa: BLE001 # no salt key or master key configured: store the value as written return value diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index 5f0a7a27bc9..4e336bf01b4 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -391,3 +391,69 @@ def test_bearer_token_uses_only_header_safe_characters(length: int): assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length + + +# --------------------------------------------------------------------------- +# Secret writers must refuse (not silently store plaintext) when the +# xsalsa20-poly1305 opt-in cannot be honored: PyNaCl missing or FIPS mode. +# --------------------------------------------------------------------------- + + +def _opt_into_xsalsa20(monkeypatch) -> None: + monkeypatch.setattr(proxy_server, "general_settings", {"encryption_algorithm": "xsalsa20-poly1305"}) + + +def _without_pynacl(monkeypatch) -> None: + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + + +def test_guardrail_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch): + """encrypt_guardrail_litellm_params used to swallow LegacyEncryptionUnavailableError + and persist the secret as plaintext; it must refuse the write instead.""" + _opt_into_xsalsa20(monkeypatch) + _without_pynacl(monkeypatch) + from litellm.proxy.guardrails.guardrail_registry import encrypt_guardrail_litellm_params + + with pytest.raises(LegacyEncryptionUnavailableError): + encrypt_guardrail_litellm_params({"api_key": "sk-plain-secret"}) + + +def test_search_tool_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch): + _opt_into_xsalsa20(monkeypatch) + _without_pynacl(monkeypatch) + from litellm.proxy.search_endpoints.search_tool_registry import _encrypted_search_tool_value + + with pytest.raises(LegacyEncryptionUnavailableError): + _encrypted_search_tool_value("sk-plain-secret") + + +def test_callback_writer_refuses_xsalsa20_optin_without_pynacl(monkeypatch): + _opt_into_xsalsa20(monkeypatch) + _without_pynacl(monkeypatch) + from litellm.proxy.common_utils.callback_utils import _encrypt_if_plaintext + + with pytest.raises(LegacyEncryptionUnavailableError): + _encrypt_if_plaintext("langfuse_public_key", "sk-plain-secret") + + +def test_guardrail_writer_refuses_xsalsa20_optin_under_fips(monkeypatch): + _opt_into_xsalsa20(monkeypatch) + monkeypatch.setenv("LITELLM_FIPS_MODE", "1") + from litellm.proxy.guardrails.guardrail_registry import encrypt_guardrail_litellm_params + + with pytest.raises(FipsModeError): + encrypt_guardrail_litellm_params({"api_key": "sk-plain-secret"}) + + +def test_secret_writers_still_encrypt_under_default_algorithm_without_pynacl(monkeypatch): + """Control: AES (the default) needs no PyNaCl, so a no-PyNaCl install still + encrypts secrets on every writer — only the refused opt-in changes behavior.""" + _use_aes(monkeypatch) + _without_pynacl(monkeypatch) + from litellm.proxy.common_utils.callback_utils import _encrypt_if_plaintext + from litellm.proxy.search_endpoints.search_tool_registry import _encrypted_search_tool_value + + assert _encrypted_search_tool_value("sk-plain-secret").startswith("v3:gcm:") + encrypted = _encrypt_if_plaintext("langfuse_public_key", "sk-plain-secret") + assert isinstance(encrypted, str) and encrypted.startswith("litellm_enc::v3:gcm:") diff --git a/tests/unit/proxy/db/test_master_key_migration.py b/tests/unit/proxy/db/test_master_key_migration.py index 4b7f61eaed1..5fce9dbfac1 100644 --- a/tests/unit/proxy/db/test_master_key_migration.py +++ b/tests/unit/proxy/db/test_master_key_migration.py @@ -648,3 +648,87 @@ async def test_guardrail_params_move_to_the_new_key_and_legacy_plaintext_rows_ar assert decrypt_if_encrypted_with(migrated_key.removeprefix("litellm_enc::"), NEW_KEY) == "guardrail-vendor-key" assert tables["LiteLLM_GuardrailsTable"][1]["litellm_params"] == legacy_params assert database.writes == [("LiteLLM_GuardrailsTable", "litellm_params", "guardrail-1")] + + +@pytest.mark.asyncio +async def test_plaintext_config_and_nested_model_leaves_do_not_block_a_pynacl_free_migration(monkeypatch): + """Without PyNaCl the count may only consider leaves the column's writers actually + encrypt. Plaintext noise — a config settings row, model tags — must not read as + values encrypted with the previous key, or the boot migration never finishes and a + FIPS image (which cannot install PyNaCl) is told forever to install the extra.""" + import sys + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + tables: Tables = { + "LiteLLM_Config": [ + {"param_name": "general_settings", "param_value": {"store_model_in_db": "true", "alerting": ["slack"]}}, + ], + "LiteLLM_ProxyModelTable": [ + { + "model_id": "model-1", + "litellm_params": { + "api_key": _encrypted("provider-key"), + "tags": ["prod"], + }, + } + ], + } + database = _FakeDatabase(tables) + + assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # only the api_key + + outcome = await migrate_from_previous_master_key( + previous_master_key=PREVIOUS_KEY, + master_key=NEW_KEY, + salt_key_is_set=False, + database=database, + log=lambda message: None, + ) + + # The one readable value migrated; the recount sees no previous-key value anywhere, + # so the operator is told the variable can go instead of being stuck on the extra. + assert outcome == Migrated(migrated=1, remaining=0, unreadable=0) + assert "You may now delete" in describe_outcome(outcome) + + second_boot = await migrate_from_previous_master_key( + previous_master_key=PREVIOUS_KEY, + master_key=NEW_KEY, + salt_key_is_set=False, + database=database, + log=lambda message: None, + ) + assert second_boot == NothingToMigrate.NOTHING_ENCRYPTED_WITH_PREVIOUS_KEY + assert ("LiteLLM_ProxyModelTable", "litellm_params", "model-1") in database.writes + row = tables["LiteLLM_ProxyModelTable"][0]["litellm_params"] + assert row["tags"] == ["prod"] # plaintext nested leaf untouched + assert decrypt_if_encrypted_with(str(row["api_key"]), NEW_KEY) == "provider-key" + assert tables["LiteLLM_Config"][0]["param_value"] == { + "store_model_in_db": "true", + "alerting": ["slack"], + } # plaintext config row untouched + + +@pytest.mark.asyncio +async def test_guardrail_mode_and_unprefixed_params_do_not_block_a_pynacl_free_migration(monkeypatch): + """A guardrails row's plaintext params never read as previous-key ciphertext either: + its writer marks every encrypted value, so only marked sensitive leaves count.""" + import sys + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + tables: Tables = { + "LiteLLM_GuardrailsTable": [ + { + "guardrail_id": "gr-1", + "litellm_params": { + "mode": "pre_call", + "guardrail_name": "presidio", + "api_key": "litellm_enc::" + _encrypted("guardrail-key"), + }, + } + ], + } + database = _FakeDatabase(tables) + + assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # the marked api_key only diff --git a/tests/unit/proxy/management_endpoints/test_credential_migration.py b/tests/unit/proxy/management_endpoints/test_credential_migration.py index 966e6fb0db9..8c5a4a860d9 100644 --- a/tests/unit/proxy/management_endpoints/test_credential_migration.py +++ b/tests/unit/proxy/management_endpoints/test_credential_migration.py @@ -1126,6 +1126,10 @@ def _covered_table_rig( litellm_params={ "mode": "pre_call", "api_key": CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("guardrail-secret"), + # Nested shapes the guardrails writer also encrypts: a sensitive key + # inside a nested dict, and sensitive-keyed strings inside a list. + "metadata": {"api_key": CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("nested")}, + "secret_headers": [CALLBACK_VAR_ENCRYPTED_PREFIX + _wrong_key_v3("listed")], }, ) )