diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 4d8bc2a79b2..9ebdb1fac8a 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -728,7 +728,8 @@ def encrypt_callback_vars(metadata: object) -> Any: def decrypt_callback_vars(metadata: object) -> Any: """Return a deep copy of metadata with callback_vars values decrypted. - Legacy plaintext rows pass through unchanged (decrypt failure → original). + Legacy plaintext rows pass through unchanged; a marked value that cannot be + decrypted reads as unset instead of being served as the credential. """ return _transform_callback_vars(metadata, _decrypt_or_passthrough) @@ -796,4 +797,8 @@ def _decrypt_or_passthrough(key: str, value: object) -> object: return value inner: Final = value[len(_CALLBACK_VAR_ENCRYPTED_PREFIX) :] decrypted: Final = decrypt_value_helper(value=inner, key=key, exception_type="debug", return_original_value=False) - return decrypted if decrypted is not None else value + if decrypted is not None: + return decrypted + # Marked-but-undecryptable (wrong key, or legacy ciphertext PyNaCl is missing for): + # hand the provider None — never the ciphertext blob itself as if it were the key. + return None diff --git a/litellm/proxy/db/master_key_migration.py b/litellm/proxy/db/master_key_migration.py index 441b05792d3..1c65f02aa91 100644 --- a/litellm/proxy/db/master_key_migration.py +++ b/litellm/proxy/db/master_key_migration.py @@ -134,6 +134,12 @@ _LITELLM_PARAMS_SECRET_COLUMNS: Final = frozenset( ) ) _ENV_VARS_CONFIG_ROW: Final = "environment_variables" +# Config rows whose writers encrypt selected top-level fields (the same fields the +# credential-migration walkers re-encrypt); the rows' other fields are plaintext metadata. +_CONFIG_ROW_SENSITIVE_FIELDS: Final[Mapping[str, tuple[str, ...]]] = { + "vantage_settings": ("api_key", "integration_token"), + "cloudzero_settings": ("api_key",), +} def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, stored: JsonValue) -> Iterator[str]: @@ -147,9 +153,22 @@ def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, st selection the credential-migration scanner applies. Columns without a known writer shape keep the conservative every-string-leaf walk. The callback-encryption marker is stripped exactly as ``replace_ciphertexts`` strips it before the decrypt attempt.""" - if secret_column.table == "LiteLLM_Config" and _PRIMARY_KEY.validate_python(primary_key) != _ENV_VARS_CONFIG_ROW: - # Only the environment_variables row's values are ever encrypted; every other - # config row (general_settings, …) stores plaintext by construction. + if secret_column.table == "LiteLLM_Config": + param_name: Final[str] = _PRIMARY_KEY.validate_python(primary_key) + if param_name == _ENV_VARS_CONFIG_ROW: + yield from _string_leaves(stored) + return + sensitive_fields: Final[tuple[str, ...] | None] = _CONFIG_ROW_SENSITIVE_FIELDS.get(param_name) + if sensitive_fields is None: + # Every other config row (general_settings, …) stores plaintext by + # construction — counting its strings would read them as ciphertext. + return + settings: Final[JsonValue] = stored + if isinstance(settings, dict): + for field in sensitive_fields: + value = settings.get(field) + if isinstance(value, str): + yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) return if (secret_column.table, secret_column.column) in _LITELLM_PARAMS_SECRET_COLUMNS: params: Final[JsonValue] = stored diff --git a/tests/unit/proxy/common_utils/test_callback_utils.py b/tests/unit/proxy/common_utils/test_callback_utils.py index ecb2375d495..ec3da28d383 100644 --- a/tests/unit/proxy/common_utils/test_callback_utils.py +++ b/tests/unit/proxy/common_utils/test_callback_utils.py @@ -26,6 +26,7 @@ from litellm.proxy.common_utils.callback_utils import ( sanitize_openai_provider_metadata, strip_callback_config, ) +from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.utils import ProxyLogging from litellm.types.guardrails import GuardrailEventHooks @@ -154,9 +155,7 @@ def test_add_policy_to_applied_policies_header_uses_litellm_metadata_bucket(): "litellm_metadata": {}, } - add_policy_to_applied_policies_header( - request_data=request_data, policy_name="global-baseline" - ) + add_policy_to_applied_policies_header(request_data=request_data, policy_name="global-baseline") assert request_data["litellm_metadata"]["applied_policies"] == ["global-baseline"] assert "applied_policies" not in request_data.get("metadata", {}) @@ -311,9 +310,7 @@ def test_initialize_callbacks_on_proxy_instantiates_compression_interception( lambda litellm_settings, callback_specific_params: dummy_callback, ) - original_callbacks = ( - list(litellm.callbacks) if isinstance(litellm.callbacks, list) else [] - ) + original_callbacks = list(litellm.callbacks) if isinstance(litellm.callbacks, list) else [] litellm.callbacks = [] try: initialize_callbacks_on_proxy( @@ -366,20 +363,11 @@ def test_encrypt_callback_vars_round_trip(monkeypatch): enc_vars = encrypted["logging"][0]["callback_vars"] assert enc_vars["langfuse_secret_key"] != "sk-lf-secret" assert enc_vars["langfuse_public_key"] != "pk-lf-public" - assert ( - encrypted["callback_settings"]["callback_vars"]["langsmith_api_key"] - != "ls-api-key" - ) + assert encrypted["callback_settings"]["callback_vars"]["langsmith_api_key"] != "ls-api-key" decrypted = decrypt_callback_vars(encrypted) - assert ( - decrypted["logging"][0]["callback_vars"] - == original["logging"][0]["callback_vars"] - ) - assert ( - decrypted["callback_settings"]["callback_vars"] - == original["callback_settings"]["callback_vars"] - ) + assert decrypted["logging"][0]["callback_vars"] == original["logging"][0]["callback_vars"] + assert decrypted["callback_settings"]["callback_vars"] == original["callback_settings"]["callback_vars"] def test_encrypt_callback_vars_is_idempotent(monkeypatch): @@ -402,10 +390,7 @@ def test_decrypt_callback_vars_passes_through_legacy_plaintext(monkeypatch): plaintext = _sample_metadata() decrypted = decrypt_callback_vars(plaintext) # legacy rows decrypt-fail and fall through unchanged - assert ( - decrypted["logging"][0]["callback_vars"]["langfuse_secret_key"] - == "sk-lf-secret" - ) + assert decrypted["logging"][0]["callback_vars"]["langfuse_secret_key"] == "sk-lf-secret" def test_callback_vars_helpers_handle_edge_shapes(monkeypatch): @@ -506,9 +491,7 @@ def test_initialize_callbacks_on_proxy_lakera_ignores_non_dict_callback_settings SimpleNamespace(prisma_client=None), ) - original_callbacks = ( - list(litellm.callbacks) if isinstance(litellm.callbacks, list) else [] - ) + original_callbacks = list(litellm.callbacks) if isinstance(litellm.callbacks, list) else [] litellm.callbacks = [] try: # A non-dict value must be ignored (init_params stays {}), not **-unpacked. @@ -526,9 +509,7 @@ def test_initialize_callbacks_on_proxy_lakera_ignores_non_dict_callback_settings @pytest.mark.parametrize("bad_root", [None, True]) -def test_initialize_callbacks_on_proxy_non_dict_callback_specific_params_root( - monkeypatch, bad_root -): +def test_initialize_callbacks_on_proxy_non_dict_callback_specific_params_root(monkeypatch, bad_root): """Regression: a blank `callback_settings:` key in YAML loads as None (and `callback_settings: true` as a bool); load_config forwards that value verbatim as callback_specific_params. Membership tests like @@ -545,9 +526,7 @@ def test_initialize_callbacks_on_proxy_non_dict_callback_specific_params_root( CompressionInterceptionLogger, ) - original_callbacks = ( - list(litellm.callbacks) if isinstance(litellm.callbacks, list) else [] - ) + original_callbacks = list(litellm.callbacks) if isinstance(litellm.callbacks, list) else [] litellm.callbacks = [] try: initialize_callbacks_on_proxy( @@ -557,9 +536,7 @@ def test_initialize_callbacks_on_proxy_non_dict_callback_specific_params_root( litellm_settings={}, callback_specific_params=bad_root, ) - assert any( - isinstance(c, CompressionInterceptionLogger) for c in litellm.callbacks - ) + assert any(isinstance(c, CompressionInterceptionLogger) for c in litellm.callbacks) finally: litellm.callbacks = original_callbacks @@ -611,7 +588,7 @@ def test_strip_callback_config_passes_through_non_dicts(value): _PROBE_MODULE_NAME = "custom_callback_probe" -_PROBE_MODULE_SOURCE = ''' +_PROBE_MODULE_SOURCE = """ from litellm.integrations.custom_logger import CustomLogger @@ -632,7 +609,7 @@ def log_event_fn(kwargs, response_obj, start_time, end_time): NOT_A_CALLBACK = "some-plain-string" proxy_handler_instance = FloorMaxTokens() -''' +""" @pytest.fixture @@ -646,9 +623,7 @@ def probe_config_path(tmp_path): """ (tmp_path / f"{_PROBE_MODULE_NAME}.py").write_text(_PROBE_MODULE_SOURCE) - original_callbacks = ( - list(litellm.callbacks) if isinstance(litellm.callbacks, list) else litellm.callbacks - ) + original_callbacks = list(litellm.callbacks) if isinstance(litellm.callbacks, list) else litellm.callbacks litellm.callbacks = [] ProxyLogging._callback_capabilities_cache.clear() try: @@ -674,7 +649,7 @@ def test_initialize_callbacks_on_proxy_rejects_class_valued_entry(probe_config_p silently never run the hook. Config load must fail instead.""" entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens" - with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: + with pytest.raises(ValueError, match="litellm_settings\\.callbacks entry") as exc_info: _load_callbacks([entry], probe_config_path) message = str(exc_info.value) @@ -692,12 +667,10 @@ def test_initialize_callbacks_on_proxy_rejects_class_valued_entry(probe_config_p ("NOT_A_CALLBACK", "str 'some-plain-string'"), ], ) -def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values( - probe_config_path, attribute, expected_fragment -): +def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values(probe_config_path, attribute, expected_fragment): entry = f"{_PROBE_MODULE_NAME}.{attribute}" - with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: + with pytest.raises(ValueError, match="litellm_settings\\.callbacks entry") as exc_info: _load_callbacks([entry], probe_config_path) message = str(exc_info.value) @@ -709,7 +682,7 @@ def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values( def test_initialize_callbacks_on_proxy_rejects_class_valued_non_list_value(probe_config_path): entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens" - with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: + with pytest.raises(ValueError, match="litellm_settings\\.callbacks entry") as exc_info: _load_callbacks(entry, probe_config_path) assert entry in str(exc_info.value) @@ -762,3 +735,28 @@ def test_initialize_callbacks_on_proxy_accepts_instance_non_list_value(probe_con assert len(litellm.callbacks) == 1 assert isinstance(litellm.callbacks[0], CustomLogger) + + +def test_decrypt_callback_vars_drops_a_marked_value_it_cannot_read(monkeypatch): + """A value carrying the encrypted-value marker that does not decrypt (wrong key, or + legacy ciphertext without PyNaCl) must read as unset — never be handed to the + logging provider as if the ciphertext blob were the credential.""" + import sys + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-callback-decrypt-salt") + marked = "litellm_enc::" + str(encrypt_value_helper("secret", new_encryption_key="sk-another-salt")) + + decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}}) + + assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] is None + + +def test_decrypt_callback_vars_round_trips_a_marked_value_under_the_current_key(monkeypatch): + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-callback-decrypt-salt") + marked = "litellm_enc::" + str(encrypt_value_helper("the-secret")) + + decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}}) + + assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] == "the-secret" diff --git a/tests/unit/proxy/db/test_master_key_migration.py b/tests/unit/proxy/db/test_master_key_migration.py index 5fce9dbfac1..b06623f8c1a 100644 --- a/tests/unit/proxy/db/test_master_key_migration.py +++ b/tests/unit/proxy/db/test_master_key_migration.py @@ -732,3 +732,43 @@ async def test_guardrail_mode_and_unprefixed_params_do_not_block_a_pynacl_free_m database = _FakeDatabase(tables) assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # the marked api_key only + + +@pytest.mark.asyncio +async def test_integration_settings_config_rows_count_their_encrypted_fields(monkeypatch): + """vantage_settings / cloudzero_settings config rows hold writer-encrypted fields + next to plaintext metadata; the count must see the encrypted fields (so the boot + re-encrypts them before the variable can be deleted) and ignore the metadata.""" + import sys + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + tables: Tables = { + "LiteLLM_Config": [ + { + "param_name": "vantage_settings", + "param_value": { + "api_key": _encrypted("vantage-key"), + "integration_token": _encrypted("vantage-token"), + "base_url": "https://api.vantage.sh", + }, + }, + {"param_name": "cloudzero_settings", "param_value": {"api_key": _encrypted("cz-key"), "timezone": "utc"}}, + ], + } + database = _FakeDatabase(tables) + + assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 3 + + outcome = await migrate_from_previous_master_key( + previous_master_key=PREVIOUS_KEY, + master_key=NEW_KEY, + salt_key_is_set=False, + database=database, + log=lambda message: None, + ) + + assert outcome == Migrated(migrated=3, remaining=0, unreadable=0) + assert decrypt_if_encrypted_with(tables["LiteLLM_Config"][0]["param_value"]["api_key"], NEW_KEY) == "vantage-key" + assert tables["LiteLLM_Config"][0]["param_value"]["base_url"] == "https://api.vantage.sh" + assert tables["LiteLLM_Config"][1]["param_value"]["timezone"] == "utc"