fix(proxy): drop unreadable marked callback values and count encrypted integration-settings config rows in the key migration
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

Two follow-up review findings on 4836dc11c0:

- _decrypt_or_passthrough handed the full litellm_enc:: ciphertext blob
  back as the credential when decryption failed (wrong key, or legacy
  ciphertext without PyNaCl). A marked value that cannot be read now
  decrypts to None — fail closed like credential hydration and the
  search-tool reader — while unprefixed legacy plaintext still passes
  through.

- The boot-migration leaf selection excluded every non-environment_variables
  config row, but vantage_settings and cloudzero_settings hold
  writer-encrypted fields (api_key / integration_token) next to plaintext
  metadata. With those the last old-key ciphertexts the count read zero,
  the boot reported nothing left to migrate, and the operator could delete
  LITELLM_MIGRATE_FROM_MASTER_KEY while those integration secrets stayed
  under the retired key. Those rows now count exactly their encrypted
  fields; other config rows stay excluded as plaintext-by-construction.
This commit is contained in:
yucheng-berri 2026-10-14 21:16:19 -07:00
parent 4836dc11c0
commit 90fa2944b1
4 changed files with 112 additions and 50 deletions

View file

@ -728,7 +728,8 @@ def encrypt_callback_vars(metadata: object) -> Any:
def decrypt_callback_vars(metadata: object) -> Any: def decrypt_callback_vars(metadata: object) -> Any:
"""Return a deep copy of metadata with callback_vars values decrypted. """Return a deep copy of metadata with callback_vars values decrypted.
Legacy plaintext rows pass through unchanged (decrypt failure → original). Legacy plaintext rows pass through unchanged; a marked value that cannot be
decrypted reads as unset instead of being served as the credential.
""" """
return _transform_callback_vars(metadata, _decrypt_or_passthrough) return _transform_callback_vars(metadata, _decrypt_or_passthrough)
@ -796,4 +797,8 @@ def _decrypt_or_passthrough(key: str, value: object) -> object:
return value return value
inner: Final = value[len(_CALLBACK_VAR_ENCRYPTED_PREFIX) :] inner: Final = value[len(_CALLBACK_VAR_ENCRYPTED_PREFIX) :]
decrypted: Final = decrypt_value_helper(value=inner, key=key, exception_type="debug", return_original_value=False) decrypted: Final = decrypt_value_helper(value=inner, key=key, exception_type="debug", return_original_value=False)
return decrypted if decrypted is not None else value if decrypted is not None:
return decrypted
# Marked-but-undecryptable (wrong key, or legacy ciphertext PyNaCl is missing for):
# hand the provider None — never the ciphertext blob itself as if it were the key.
return None

View file

@ -134,6 +134,12 @@ _LITELLM_PARAMS_SECRET_COLUMNS: Final = frozenset(
) )
) )
_ENV_VARS_CONFIG_ROW: Final = "environment_variables" _ENV_VARS_CONFIG_ROW: Final = "environment_variables"
# Config rows whose writers encrypt selected top-level fields (the same fields the
# credential-migration walkers re-encrypt); the rows' other fields are plaintext metadata.
_CONFIG_ROW_SENSITIVE_FIELDS: Final[Mapping[str, tuple[str, ...]]] = {
"vantage_settings": ("api_key", "integration_token"),
"cloudzero_settings": ("api_key",),
}
def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, stored: JsonValue) -> Iterator[str]: def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, stored: JsonValue) -> Iterator[str]:
@ -147,9 +153,22 @@ def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, st
selection the credential-migration scanner applies. Columns without a known writer selection the credential-migration scanner applies. Columns without a known writer
shape keep the conservative every-string-leaf walk. The callback-encryption marker is shape keep the conservative every-string-leaf walk. The callback-encryption marker is
stripped exactly as ``replace_ciphertexts`` strips it before the decrypt attempt.""" stripped exactly as ``replace_ciphertexts`` strips it before the decrypt attempt."""
if secret_column.table == "LiteLLM_Config" and _PRIMARY_KEY.validate_python(primary_key) != _ENV_VARS_CONFIG_ROW: if secret_column.table == "LiteLLM_Config":
# Only the environment_variables row's values are ever encrypted; every other param_name: Final[str] = _PRIMARY_KEY.validate_python(primary_key)
# config row (general_settings, …) stores plaintext by construction. if param_name == _ENV_VARS_CONFIG_ROW:
yield from _string_leaves(stored)
return
sensitive_fields: Final[tuple[str, ...] | None] = _CONFIG_ROW_SENSITIVE_FIELDS.get(param_name)
if sensitive_fields is None:
# Every other config row (general_settings, …) stores plaintext by
# construction — counting its strings would read them as ciphertext.
return
settings: Final[JsonValue] = stored
if isinstance(settings, dict):
for field in sensitive_fields:
value = settings.get(field)
if isinstance(value, str):
yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX)
return return
if (secret_column.table, secret_column.column) in _LITELLM_PARAMS_SECRET_COLUMNS: if (secret_column.table, secret_column.column) in _LITELLM_PARAMS_SECRET_COLUMNS:
params: Final[JsonValue] = stored params: Final[JsonValue] = stored

View file

@ -26,6 +26,7 @@ from litellm.proxy.common_utils.callback_utils import (
sanitize_openai_provider_metadata, sanitize_openai_provider_metadata,
strip_callback_config, strip_callback_config,
) )
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.utils import ProxyLogging from litellm.proxy.utils import ProxyLogging
from litellm.types.guardrails import GuardrailEventHooks from litellm.types.guardrails import GuardrailEventHooks
@ -154,9 +155,7 @@ def test_add_policy_to_applied_policies_header_uses_litellm_metadata_bucket():
"litellm_metadata": {}, "litellm_metadata": {},
} }
add_policy_to_applied_policies_header( add_policy_to_applied_policies_header(request_data=request_data, policy_name="global-baseline")
request_data=request_data, policy_name="global-baseline"
)
assert request_data["litellm_metadata"]["applied_policies"] == ["global-baseline"] assert request_data["litellm_metadata"]["applied_policies"] == ["global-baseline"]
assert "applied_policies" not in request_data.get("metadata", {}) assert "applied_policies" not in request_data.get("metadata", {})
@ -311,9 +310,7 @@ def test_initialize_callbacks_on_proxy_instantiates_compression_interception(
lambda litellm_settings, callback_specific_params: dummy_callback, lambda litellm_settings, callback_specific_params: dummy_callback,
) )
original_callbacks = ( original_callbacks = list(litellm.callbacks) if isinstance(litellm.callbacks, list) else []
list(litellm.callbacks) if isinstance(litellm.callbacks, list) else []
)
litellm.callbacks = [] litellm.callbacks = []
try: try:
initialize_callbacks_on_proxy( initialize_callbacks_on_proxy(
@ -366,20 +363,11 @@ def test_encrypt_callback_vars_round_trip(monkeypatch):
enc_vars = encrypted["logging"][0]["callback_vars"] enc_vars = encrypted["logging"][0]["callback_vars"]
assert enc_vars["langfuse_secret_key"] != "sk-lf-secret" assert enc_vars["langfuse_secret_key"] != "sk-lf-secret"
assert enc_vars["langfuse_public_key"] != "pk-lf-public" assert enc_vars["langfuse_public_key"] != "pk-lf-public"
assert ( assert encrypted["callback_settings"]["callback_vars"]["langsmith_api_key"] != "ls-api-key"
encrypted["callback_settings"]["callback_vars"]["langsmith_api_key"]
!= "ls-api-key"
)
decrypted = decrypt_callback_vars(encrypted) decrypted = decrypt_callback_vars(encrypted)
assert ( assert decrypted["logging"][0]["callback_vars"] == original["logging"][0]["callback_vars"]
decrypted["logging"][0]["callback_vars"] assert decrypted["callback_settings"]["callback_vars"] == original["callback_settings"]["callback_vars"]
== original["logging"][0]["callback_vars"]
)
assert (
decrypted["callback_settings"]["callback_vars"]
== original["callback_settings"]["callback_vars"]
)
def test_encrypt_callback_vars_is_idempotent(monkeypatch): def test_encrypt_callback_vars_is_idempotent(monkeypatch):
@ -402,10 +390,7 @@ def test_decrypt_callback_vars_passes_through_legacy_plaintext(monkeypatch):
plaintext = _sample_metadata() plaintext = _sample_metadata()
decrypted = decrypt_callback_vars(plaintext) decrypted = decrypt_callback_vars(plaintext)
# legacy rows decrypt-fail and fall through unchanged # legacy rows decrypt-fail and fall through unchanged
assert ( assert decrypted["logging"][0]["callback_vars"]["langfuse_secret_key"] == "sk-lf-secret"
decrypted["logging"][0]["callback_vars"]["langfuse_secret_key"]
== "sk-lf-secret"
)
def test_callback_vars_helpers_handle_edge_shapes(monkeypatch): def test_callback_vars_helpers_handle_edge_shapes(monkeypatch):
@ -506,9 +491,7 @@ def test_initialize_callbacks_on_proxy_lakera_ignores_non_dict_callback_settings
SimpleNamespace(prisma_client=None), SimpleNamespace(prisma_client=None),
) )
original_callbacks = ( original_callbacks = list(litellm.callbacks) if isinstance(litellm.callbacks, list) else []
list(litellm.callbacks) if isinstance(litellm.callbacks, list) else []
)
litellm.callbacks = [] litellm.callbacks = []
try: try:
# A non-dict value must be ignored (init_params stays {}), not **-unpacked. # A non-dict value must be ignored (init_params stays {}), not **-unpacked.
@ -526,9 +509,7 @@ def test_initialize_callbacks_on_proxy_lakera_ignores_non_dict_callback_settings
@pytest.mark.parametrize("bad_root", [None, True]) @pytest.mark.parametrize("bad_root", [None, True])
def test_initialize_callbacks_on_proxy_non_dict_callback_specific_params_root( def test_initialize_callbacks_on_proxy_non_dict_callback_specific_params_root(monkeypatch, bad_root):
monkeypatch, bad_root
):
"""Regression: a blank `callback_settings:` key in YAML loads as None (and """Regression: a blank `callback_settings:` key in YAML loads as None (and
`callback_settings: true` as a bool); load_config forwards that value `callback_settings: true` as a bool); load_config forwards that value
verbatim as callback_specific_params. Membership tests like verbatim as callback_specific_params. Membership tests like
@ -545,9 +526,7 @@ def test_initialize_callbacks_on_proxy_non_dict_callback_specific_params_root(
CompressionInterceptionLogger, CompressionInterceptionLogger,
) )
original_callbacks = ( original_callbacks = list(litellm.callbacks) if isinstance(litellm.callbacks, list) else []
list(litellm.callbacks) if isinstance(litellm.callbacks, list) else []
)
litellm.callbacks = [] litellm.callbacks = []
try: try:
initialize_callbacks_on_proxy( initialize_callbacks_on_proxy(
@ -557,9 +536,7 @@ def test_initialize_callbacks_on_proxy_non_dict_callback_specific_params_root(
litellm_settings={}, litellm_settings={},
callback_specific_params=bad_root, callback_specific_params=bad_root,
) )
assert any( assert any(isinstance(c, CompressionInterceptionLogger) for c in litellm.callbacks)
isinstance(c, CompressionInterceptionLogger) for c in litellm.callbacks
)
finally: finally:
litellm.callbacks = original_callbacks litellm.callbacks = original_callbacks
@ -611,7 +588,7 @@ def test_strip_callback_config_passes_through_non_dicts(value):
_PROBE_MODULE_NAME = "custom_callback_probe" _PROBE_MODULE_NAME = "custom_callback_probe"
_PROBE_MODULE_SOURCE = ''' _PROBE_MODULE_SOURCE = """
from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_logger import CustomLogger
@ -632,7 +609,7 @@ def log_event_fn(kwargs, response_obj, start_time, end_time):
NOT_A_CALLBACK = "some-plain-string" NOT_A_CALLBACK = "some-plain-string"
proxy_handler_instance = FloorMaxTokens() proxy_handler_instance = FloorMaxTokens()
''' """
@pytest.fixture @pytest.fixture
@ -646,9 +623,7 @@ def probe_config_path(tmp_path):
""" """
(tmp_path / f"{_PROBE_MODULE_NAME}.py").write_text(_PROBE_MODULE_SOURCE) (tmp_path / f"{_PROBE_MODULE_NAME}.py").write_text(_PROBE_MODULE_SOURCE)
original_callbacks = ( original_callbacks = list(litellm.callbacks) if isinstance(litellm.callbacks, list) else litellm.callbacks
list(litellm.callbacks) if isinstance(litellm.callbacks, list) else litellm.callbacks
)
litellm.callbacks = [] litellm.callbacks = []
ProxyLogging._callback_capabilities_cache.clear() ProxyLogging._callback_capabilities_cache.clear()
try: try:
@ -674,7 +649,7 @@ def test_initialize_callbacks_on_proxy_rejects_class_valued_entry(probe_config_p
silently never run the hook. Config load must fail instead.""" silently never run the hook. Config load must fail instead."""
entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens" entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens"
with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: with pytest.raises(ValueError, match="litellm_settings\\.callbacks entry") as exc_info:
_load_callbacks([entry], probe_config_path) _load_callbacks([entry], probe_config_path)
message = str(exc_info.value) message = str(exc_info.value)
@ -692,12 +667,10 @@ def test_initialize_callbacks_on_proxy_rejects_class_valued_entry(probe_config_p
("NOT_A_CALLBACK", "str 'some-plain-string'"), ("NOT_A_CALLBACK", "str 'some-plain-string'"),
], ],
) )
def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values( def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values(probe_config_path, attribute, expected_fragment):
probe_config_path, attribute, expected_fragment
):
entry = f"{_PROBE_MODULE_NAME}.{attribute}" entry = f"{_PROBE_MODULE_NAME}.{attribute}"
with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: with pytest.raises(ValueError, match="litellm_settings\\.callbacks entry") as exc_info:
_load_callbacks([entry], probe_config_path) _load_callbacks([entry], probe_config_path)
message = str(exc_info.value) message = str(exc_info.value)
@ -709,7 +682,7 @@ def test_initialize_callbacks_on_proxy_rejects_non_dispatchable_values(
def test_initialize_callbacks_on_proxy_rejects_class_valued_non_list_value(probe_config_path): def test_initialize_callbacks_on_proxy_rejects_class_valued_non_list_value(probe_config_path):
entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens" entry = f"{_PROBE_MODULE_NAME}.FloorMaxTokens"
with pytest.raises(ValueError, match='litellm_settings\\.callbacks entry') as exc_info: with pytest.raises(ValueError, match="litellm_settings\\.callbacks entry") as exc_info:
_load_callbacks(entry, probe_config_path) _load_callbacks(entry, probe_config_path)
assert entry in str(exc_info.value) assert entry in str(exc_info.value)
@ -762,3 +735,28 @@ def test_initialize_callbacks_on_proxy_accepts_instance_non_list_value(probe_con
assert len(litellm.callbacks) == 1 assert len(litellm.callbacks) == 1
assert isinstance(litellm.callbacks[0], CustomLogger) assert isinstance(litellm.callbacks[0], CustomLogger)
def test_decrypt_callback_vars_drops_a_marked_value_it_cannot_read(monkeypatch):
"""A value carrying the encrypted-value marker that does not decrypt (wrong key, or
legacy ciphertext without PyNaCl) must read as unset — never be handed to the
logging provider as if the ciphertext blob were the credential."""
import sys
monkeypatch.setitem(sys.modules, "nacl", None)
monkeypatch.setitem(sys.modules, "nacl.secret", None)
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-callback-decrypt-salt")
marked = "litellm_enc::" + str(encrypt_value_helper("secret", new_encryption_key="sk-another-salt"))
decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}})
assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] is None
def test_decrypt_callback_vars_round_trips_a_marked_value_under_the_current_key(monkeypatch):
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-callback-decrypt-salt")
marked = "litellm_enc::" + str(encrypt_value_helper("the-secret"))
decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}})
assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] == "the-secret"

View file

@ -732,3 +732,43 @@ async def test_guardrail_mode_and_unprefixed_params_do_not_block_a_pynacl_free_m
database = _FakeDatabase(tables) database = _FakeDatabase(tables)
assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # the marked api_key only assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # the marked api_key only
@pytest.mark.asyncio
async def test_integration_settings_config_rows_count_their_encrypted_fields(monkeypatch):
"""vantage_settings / cloudzero_settings config rows hold writer-encrypted fields
next to plaintext metadata; the count must see the encrypted fields (so the boot
re-encrypts them before the variable can be deleted) and ignore the metadata."""
import sys
monkeypatch.setitem(sys.modules, "nacl", None)
monkeypatch.setitem(sys.modules, "nacl.secret", None)
tables: Tables = {
"LiteLLM_Config": [
{
"param_name": "vantage_settings",
"param_value": {
"api_key": _encrypted("vantage-key"),
"integration_token": _encrypted("vantage-token"),
"base_url": "https://api.vantage.sh",
},
},
{"param_name": "cloudzero_settings", "param_value": {"api_key": _encrypted("cz-key"), "timezone": "utc"}},
],
}
database = _FakeDatabase(tables)
assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 3
outcome = await migrate_from_previous_master_key(
previous_master_key=PREVIOUS_KEY,
master_key=NEW_KEY,
salt_key_is_set=False,
database=database,
log=lambda message: None,
)
assert outcome == Migrated(migrated=3, remaining=0, unreadable=0)
assert decrypt_if_encrypted_with(tables["LiteLLM_Config"][0]["param_value"]["api_key"], NEW_KEY) == "vantage-key"
assert tables["LiteLLM_Config"][0]["param_value"]["base_url"] == "https://api.vantage.sh"
assert tables["LiteLLM_Config"][1]["param_value"]["timezone"] == "utc"