diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 9ebdb1fac8a..0b99cc5c423 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -728,8 +728,9 @@ def encrypt_callback_vars(metadata: object) -> Any: def decrypt_callback_vars(metadata: object) -> Any: """Return a deep copy of metadata with callback_vars values decrypted. - Legacy plaintext rows pass through unchanged; a marked value that cannot be - decrypted reads as unset instead of being served as the credential. + Legacy plaintext rows and marked-but-undecryptable values pass through unchanged — + the migration round-trips this transform, so the stored bytes must survive it; + serving paths drop still-marked values instead of using them as credentials. """ return _transform_callback_vars(metadata, _decrypt_or_passthrough) @@ -800,5 +801,7 @@ def _decrypt_or_passthrough(key: str, value: object) -> object: if decrypted is not None: return decrypted # Marked-but-undecryptable (wrong key, or legacy ciphertext PyNaCl is missing for): - # hand the provider None — never the ciphertext blob itself as if it were the key. - return None + # keep the stored value byte-identical — the migration round-trips this transform, + # so nulling the field would destroy the only copy of the ciphertext. Consumers + # that hand callback vars to a provider drop still-marked values as unset. + return value diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 5ea4e85a730..ca92d00c9c1 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -70,6 +70,7 @@ from litellm.proxy._types import ( from litellm.proxy.auth.auth_utils import get_request_route from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.common_utils.callback_utils import ( + CALLBACK_VAR_ENCRYPTED_PREFIX, decrypt_callback_vars, get_metadata_variable_name_from_kwargs, strip_callback_config, @@ -989,6 +990,10 @@ def convert_key_logging_metadata_to_callback( continue if var.startswith("signoz_") and data.callback_name != "signoz": continue + if value.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX): + # At-rest ciphertext this install cannot read: never coerce the blob into the + # credential — the variable reads as unset. + continue if team_callback_settings_obj.callback_vars is None: team_callback_settings_obj.callback_vars = {} team_callback_settings_obj.callback_vars[var] = str(value) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index c2d7033c06c..6ce345bf329 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -158,6 +158,13 @@ def _resolve_team_callbacks(team_metadata: object) -> TeamCallbackMetadata: callback = get_validated_callback_metadata(item=entry, source="team-level read") if callback is None: continue + # The conversion drops still-marked (unreadable) values so they are never + # served as credentials; this admin read still shows the field, masked. + for var, value in callback.callback_vars.items(): + if value.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX): + if resolved.callback_vars is None: + resolved.callback_vars = {} + resolved.callback_vars.setdefault(var, _CALLBACK_VARS_REDACTED) resolved = convert_key_logging_metadata_to_callback(data=callback, team_callback_settings_obj=resolved) else: callback_settings: Final = decrypted.get("callback_settings") diff --git a/tests/unit/proxy/common_utils/test_callback_utils.py b/tests/unit/proxy/common_utils/test_callback_utils.py index ec3da28d383..8d70f9504e3 100644 --- a/tests/unit/proxy/common_utils/test_callback_utils.py +++ b/tests/unit/proxy/common_utils/test_callback_utils.py @@ -737,10 +737,11 @@ def test_initialize_callbacks_on_proxy_accepts_instance_non_list_value(probe_con assert isinstance(litellm.callbacks[0], CustomLogger) -def test_decrypt_callback_vars_drops_a_marked_value_it_cannot_read(monkeypatch): +def test_decrypt_callback_vars_preserves_a_marked_value_it_cannot_read(monkeypatch): """A value carrying the encrypted-value marker that does not decrypt (wrong key, or - legacy ciphertext without PyNaCl) must read as unset — never be handed to the - logging provider as if the ciphertext blob were the credential.""" + legacy ciphertext without PyNaCl) survives the transform byte-identical — the + migration round-trips this decrypt and nulling the field would destroy the only copy + of the ciphertext. The serving paths drop still-marked values as unset instead.""" import sys monkeypatch.setitem(sys.modules, "nacl", None) @@ -750,7 +751,26 @@ def test_decrypt_callback_vars_drops_a_marked_value_it_cannot_read(monkeypatch): decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}}) - assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] is None + assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] == marked + + +def test_key_logging_conversion_drops_a_marked_value_it_cannot_read(monkeypatch): + """convert_key_logging_metadata_to_callback must never hand a still-marked + (unreadable) ciphertext to the logging provider as if it were the credential — the + variable reads as unset. The stored value itself stays byte-identical.""" + from litellm.proxy._types import AddTeamCallback, TeamCallbackMetadata + from litellm.proxy.litellm_pre_call_utils import convert_key_logging_metadata_to_callback + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-callback-decrypt-salt") + marked = "litellm_enc::" + str(encrypt_value_helper("secret", new_encryption_key="sk-another-salt")) + + team_callback = AddTeamCallback( + callback_name="langfuse", + callback_vars={"langfuse_secret_key": marked, "langfuse_environment": "production"}, + ) + + converted = convert_key_logging_metadata_to_callback(team_callback, TeamCallbackMetadata()) + assert converted.callback_vars == {"langfuse_environment": "production"} def test_decrypt_callback_vars_round_trips_a_marked_value_under_the_current_key(monkeypatch):