fix(proxy): never serve an unreadable marked callback value as the credential

Follow-up to the two Bugbot regressions on 90fa2944b1: dropping the value
inside _decrypt_or_passthrough null'd the field through the migration's
decrypt→re-encrypt round-trip (destroying the only copy of the ciphertext)
and str()-coercion turned the None into the literal credential "None".

_decrypt_or_passthrough keeps passing the marked value through byte-identical
(the migration round-trips it), and the serving path drops it instead:
convert_key_logging_metadata_to_callback skips vars that still carry the
encrypted-value marker, so they read as unset for providers. The team
callbacks admin read still shows those fields, masked, exactly as before.
This commit is contained in:
yucheng-berri 2026-10-14 21:38:51 -07:00
parent 90fa2944b1
commit efe26bb182
4 changed files with 43 additions and 8 deletions

View file

@ -728,8 +728,9 @@ def encrypt_callback_vars(metadata: object) -> Any:
def decrypt_callback_vars(metadata: object) -> Any: def decrypt_callback_vars(metadata: object) -> Any:
"""Return a deep copy of metadata with callback_vars values decrypted. """Return a deep copy of metadata with callback_vars values decrypted.
Legacy plaintext rows pass through unchanged; a marked value that cannot be Legacy plaintext rows and marked-but-undecryptable values pass through unchanged —
decrypted reads as unset instead of being served as the credential. the migration round-trips this transform, so the stored bytes must survive it;
serving paths drop still-marked values instead of using them as credentials.
""" """
return _transform_callback_vars(metadata, _decrypt_or_passthrough) return _transform_callback_vars(metadata, _decrypt_or_passthrough)
@ -800,5 +801,7 @@ def _decrypt_or_passthrough(key: str, value: object) -> object:
if decrypted is not None: if decrypted is not None:
return decrypted return decrypted
# Marked-but-undecryptable (wrong key, or legacy ciphertext PyNaCl is missing for): # Marked-but-undecryptable (wrong key, or legacy ciphertext PyNaCl is missing for):
# hand the provider None — never the ciphertext blob itself as if it were the key. # keep the stored value byte-identical — the migration round-trips this transform,
return None # so nulling the field would destroy the only copy of the ciphertext. Consumers
# that hand callback vars to a provider drop still-marked values as unset.
return value

View file

@ -70,6 +70,7 @@ from litellm.proxy._types import (
from litellm.proxy.auth.auth_utils import get_request_route from litellm.proxy.auth.auth_utils import get_request_route
from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.callback_utils import ( from litellm.proxy.common_utils.callback_utils import (
CALLBACK_VAR_ENCRYPTED_PREFIX,
decrypt_callback_vars, decrypt_callback_vars,
get_metadata_variable_name_from_kwargs, get_metadata_variable_name_from_kwargs,
strip_callback_config, strip_callback_config,
@ -989,6 +990,10 @@ def convert_key_logging_metadata_to_callback(
continue continue
if var.startswith("signoz_") and data.callback_name != "signoz": if var.startswith("signoz_") and data.callback_name != "signoz":
continue continue
if value.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX):
# At-rest ciphertext this install cannot read: never coerce the blob into the
# credential — the variable reads as unset.
continue
if team_callback_settings_obj.callback_vars is None: if team_callback_settings_obj.callback_vars is None:
team_callback_settings_obj.callback_vars = {} team_callback_settings_obj.callback_vars = {}
team_callback_settings_obj.callback_vars[var] = str(value) team_callback_settings_obj.callback_vars[var] = str(value)

View file

@ -158,6 +158,13 @@ def _resolve_team_callbacks(team_metadata: object) -> TeamCallbackMetadata:
callback = get_validated_callback_metadata(item=entry, source="team-level read") callback = get_validated_callback_metadata(item=entry, source="team-level read")
if callback is None: if callback is None:
continue continue
# The conversion drops still-marked (unreadable) values so they are never
# served as credentials; this admin read still shows the field, masked.
for var, value in callback.callback_vars.items():
if value.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX):
if resolved.callback_vars is None:
resolved.callback_vars = {}
resolved.callback_vars.setdefault(var, _CALLBACK_VARS_REDACTED)
resolved = convert_key_logging_metadata_to_callback(data=callback, team_callback_settings_obj=resolved) resolved = convert_key_logging_metadata_to_callback(data=callback, team_callback_settings_obj=resolved)
else: else:
callback_settings: Final = decrypted.get("callback_settings") callback_settings: Final = decrypted.get("callback_settings")

View file

@ -737,10 +737,11 @@ def test_initialize_callbacks_on_proxy_accepts_instance_non_list_value(probe_con
assert isinstance(litellm.callbacks[0], CustomLogger) assert isinstance(litellm.callbacks[0], CustomLogger)
def test_decrypt_callback_vars_drops_a_marked_value_it_cannot_read(monkeypatch): def test_decrypt_callback_vars_preserves_a_marked_value_it_cannot_read(monkeypatch):
"""A value carrying the encrypted-value marker that does not decrypt (wrong key, or """A value carrying the encrypted-value marker that does not decrypt (wrong key, or
legacy ciphertext without PyNaCl) must read as unset — never be handed to the legacy ciphertext without PyNaCl) survives the transform byte-identical — the
logging provider as if the ciphertext blob were the credential.""" migration round-trips this decrypt and nulling the field would destroy the only copy
of the ciphertext. The serving paths drop still-marked values as unset instead."""
import sys import sys
monkeypatch.setitem(sys.modules, "nacl", None) monkeypatch.setitem(sys.modules, "nacl", None)
@ -750,7 +751,26 @@ def test_decrypt_callback_vars_drops_a_marked_value_it_cannot_read(monkeypatch):
decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}}) decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}})
assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] is None assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] == marked
def test_key_logging_conversion_drops_a_marked_value_it_cannot_read(monkeypatch):
"""convert_key_logging_metadata_to_callback must never hand a still-marked
(unreadable) ciphertext to the logging provider as if it were the credential — the
variable reads as unset. The stored value itself stays byte-identical."""
from litellm.proxy._types import AddTeamCallback, TeamCallbackMetadata
from litellm.proxy.litellm_pre_call_utils import convert_key_logging_metadata_to_callback
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-callback-decrypt-salt")
marked = "litellm_enc::" + str(encrypt_value_helper("secret", new_encryption_key="sk-another-salt"))
team_callback = AddTeamCallback(
callback_name="langfuse",
callback_vars={"langfuse_secret_key": marked, "langfuse_environment": "production"},
)
converted = convert_key_logging_metadata_to_callback(team_callback, TeamCallbackMetadata())
assert converted.callback_vars == {"langfuse_environment": "production"}
def test_decrypt_callback_vars_round_trips_a_marked_value_under_the_current_key(monkeypatch): def test_decrypt_callback_vars_round_trips_a_marked_value_under_the_current_key(monkeypatch):