fix(proxy): drop unreadable marked callback vars from every serving path and count MCP writer-encrypted leaves in the key migration

Two review findings on efe26bb182:

- The marked-value drop only covered convert_key_logging_metadata_to_callback;
  the deprecated metadata['callback_settings'] branch and the OTEL tenant
  destination still handed the litellm_enc:: blob onward as the live
  credential. Both now filter through _without_unreadable_callback_vars, so
  the variable reads as unset while the stored value stays untouched.

- The boot-migration leaf selection still fell through to every-string-leaf
  for the MCP tables, so credentials metadata (aws_region_name, scopes) and
  user-scope env-var placeholders counted as old-key ciphertext without
  PyNaCl and the migration never converged on a FIPS image. The MCP
  credentials columns now count only MCP_CREDENTIAL_SECRET_FIELDS and
  env_vars only global-scope values, matching the credential-migration
  scanner's selection.
This commit is contained in:
yucheng-berri 2026-10-14 22:06:30 -07:00
parent efe26bb182
commit 543f570810
4 changed files with 132 additions and 5 deletions

View file

@ -134,6 +134,7 @@ _LITELLM_PARAMS_SECRET_COLUMNS: Final = frozenset(
)
)
_ENV_VARS_CONFIG_ROW: Final = "environment_variables"
_MCP_CREDENTIAL_TABLES: Final = frozenset(("LiteLLM_MCPServerTable", "LiteLLM_MCPServerOAuthClient"))
# Config rows whose writers encrypt selected top-level fields (the same fields the
# credential-migration walkers re-encrypt); the rows' other fields are plaintext metadata.
_CONFIG_ROW_SENSITIVE_FIELDS: Final[Mapping[str, tuple[str, ...]]] = {
@ -180,6 +181,31 @@ def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, st
if secret_column.table == "LiteLLM_GuardrailsTable":
yield from _sensitive_prefixed_leaves(stored)
return
if secret_column.column == "credentials" and secret_column.table in _MCP_CREDENTIAL_TABLES:
# The MCP credentials writer encrypts only its secret fields; region / service
# names and scopes next to them are stored plaintext by construction.
from litellm.proxy._experimental.mcp_server.db import MCP_CREDENTIAL_SECRET_FIELDS
blob: Final[JsonValue] = stored
if isinstance(blob, dict):
for field in MCP_CREDENTIAL_SECRET_FIELDS:
value = blob.get(field)
if isinstance(value, str):
yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX)
return
if secret_column.table == "LiteLLM_MCPServerTable" and secret_column.column == "env_vars":
# Only global-scope entries carry an encrypted value; user-scope placeholders
# hold a name and scope in plaintext by construction.
from litellm.proxy._experimental.mcp_server.db import is_global_env_var_scope
entries: Final[JsonValue] = stored
if isinstance(entries, list):
for entry in entries:
if isinstance(entry, dict) and is_global_env_var_scope(entry.get("scope")):
value = entry.get("value")
if isinstance(value, str):
yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX)
return
yield from _string_leaves(stored)

View file

@ -1084,6 +1084,11 @@ def get_dynamic_logging_metadata(
team_metadata: Final = decrypt_callback_vars(user_api_key_dict.team_metadata)
callback_settings: Final = team_metadata.get("callback_settings", None) or {}
callback_settings_obj = TeamCallbackMetadata(**callback_settings)
# Drop vars that still carry the marker so an unreadable ciphertext is never
# handed onward as the live credential; the variable reads as unset.
callback_settings_obj.callback_vars = cast( # cast-ok: filtering keeps the model's str values
"dict[str, str]", _without_unreadable_callback_vars(callback_settings_obj.callback_vars or {})
)
verbose_proxy_logger.debug("Team callback settings activated: %s", callback_settings_obj)
#########################################################################################
# Enter here when configured on the config.yaml file.
@ -1200,6 +1205,30 @@ def resolve_tenant_otel_destinations(
)
def _without_unreadable_callback_vars(
callback_vars: Mapping[str, object],
) -> dict[str, object]: # mutable-ok: a fresh plain mapping the callers merge into settings
"""The vars without values that still carry the encrypted-value marker.
A marked value this install cannot decrypt (wrong key, or legacy ciphertext without
PyNaCl) is never a usable credential: handing it onward — into request kwargs or OTEL
headers — would treat the ciphertext blob as the live secret. The variable reads as
unset instead; the stored value itself is untouched.
"""
usable: Final[dict[str, object]] = { # mutable-ok: filtered copy handed to the caller
var: value
for var, value in callback_vars.items()
if not (isinstance(value, str) and value.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX))
}
if len(usable) != len(callback_vars):
verbose_proxy_logger.warning(
"Ignoring %d callback variable(s) still holding unreadable encrypted values: %s",
len(callback_vars) - len(usable),
sorted(set(callback_vars) - set(usable)),
)
return usable
def _tenant_destination(
name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None
) -> "OtelDestination | None":
@ -1207,10 +1236,15 @@ def _tenant_destination(
from litellm.integrations.otel.presets.destinations import destination_for
callback_vars: Final = MappingProxyType(
dict(
itertools.chain.from_iterable(
callback.callback_vars.items() for callback in callbacks if callback.callback_name == name
)
cast( # cast-ok: filtering AddTeamCallback.callback_vars keeps its dict[str, str]
"dict[str, str]",
_without_unreadable_callback_vars(
dict(
itertools.chain.from_iterable(
callback.callback_vars.items() for callback in callbacks if callback.callback_name == name
)
)
),
)
)
return destination_for(

View file

@ -3,7 +3,7 @@ import json
import sys
from types import ModuleType, SimpleNamespace
from typing import Final
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import pytest
@ -780,3 +780,29 @@ def test_decrypt_callback_vars_round_trips_a_marked_value_under_the_current_key(
decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}})
assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] == "the-secret"
def test_deprecated_callback_settings_branch_drops_a_marked_value(monkeypatch):
"""The deprecated metadata['callback_settings'] shape must not hand a still-marked
(unreadable) ciphertext to a provider as the live credential either."""
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import get_dynamic_logging_metadata
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-callback-decrypt-salt")
marked = "litellm_enc::" + str(encrypt_value_helper("secret", new_encryption_key="sk-another-salt"))
team_metadata: Final = {
"callback_settings": {
"success_callback": ["langfuse"],
"callback_vars": {"langfuse_secret_key": marked, "langfuse_environment": "production"},
}
}
auth: Final = UserAPIKeyAuth(
api_key="sk-test",
team_id="team-1",
user_role="proxy_admin",
team_metadata=team_metadata,
)
resolved = get_dynamic_logging_metadata(user_api_key_dict=auth, proxy_config=MagicMock())
assert resolved.callback_vars == {"langfuse_environment": "production"}

View file

@ -772,3 +772,44 @@ async def test_integration_settings_config_rows_count_their_encrypted_fields(mon
assert decrypt_if_encrypted_with(tables["LiteLLM_Config"][0]["param_value"]["api_key"], NEW_KEY) == "vantage-key"
assert tables["LiteLLM_Config"][0]["param_value"]["base_url"] == "https://api.vantage.sh"
assert tables["LiteLLM_Config"][1]["param_value"]["timezone"] == "utc"
@pytest.mark.asyncio
async def test_mcp_metadata_and_user_scope_env_vars_do_not_block_a_pynacl_free_migration(monkeypatch):
"""MCP credentials hold plaintext metadata (region, scopes) next to the secret fields,
and env_vars hold plaintext user-scope placeholders: only writer-encrypted leaves
count, so a no-PyNaCl install still converges."""
import sys
monkeypatch.setitem(sys.modules, "nacl", None)
monkeypatch.setitem(sys.modules, "nacl.secret", None)
tables: Tables = {
"LiteLLM_MCPServerTable": [
{
"server_id": "mcp-1",
"credentials": {
"auth_value": _encrypted("mcp-secret"),
"aws_region_name": "us-east-1",
"scopes": ["read"],
},
"env_vars": [{"name": "API_TOKEN", "scope": "user"}],
}
],
}
database = _FakeDatabase(tables)
assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # only auth_value
outcome = await migrate_from_previous_master_key(
previous_master_key=PREVIOUS_KEY,
master_key=NEW_KEY,
salt_key_is_set=False,
database=database,
log=lambda message: None,
)
assert outcome == Migrated(migrated=1, remaining=0, unreadable=0)
credentials = tables["LiteLLM_MCPServerTable"][0]["credentials"]
assert credentials["aws_region_name"] == "us-east-1"
assert credentials["scopes"] == ["read"]
assert tables["LiteLLM_MCPServerTable"][0]["env_vars"] == [{"name": "API_TOKEN", "scope": "user"}]