diff --git a/litellm/proxy/db/master_key_migration.py b/litellm/proxy/db/master_key_migration.py index 1c65f02aa91..a23d078ecd4 100644 --- a/litellm/proxy/db/master_key_migration.py +++ b/litellm/proxy/db/master_key_migration.py @@ -134,6 +134,7 @@ _LITELLM_PARAMS_SECRET_COLUMNS: Final = frozenset( ) ) _ENV_VARS_CONFIG_ROW: Final = "environment_variables" +_MCP_CREDENTIAL_TABLES: Final = frozenset(("LiteLLM_MCPServerTable", "LiteLLM_MCPServerOAuthClient")) # Config rows whose writers encrypt selected top-level fields (the same fields the # credential-migration walkers re-encrypt); the rows' other fields are plaintext metadata. _CONFIG_ROW_SENSITIVE_FIELDS: Final[Mapping[str, tuple[str, ...]]] = { @@ -180,6 +181,31 @@ def _counted_secret_leaves(secret_column: _SecretColumn, primary_key: object, st if secret_column.table == "LiteLLM_GuardrailsTable": yield from _sensitive_prefixed_leaves(stored) return + if secret_column.column == "credentials" and secret_column.table in _MCP_CREDENTIAL_TABLES: + # The MCP credentials writer encrypts only its secret fields; region / service + # names and scopes next to them are stored plaintext by construction. + from litellm.proxy._experimental.mcp_server.db import MCP_CREDENTIAL_SECRET_FIELDS + + blob: Final[JsonValue] = stored + if isinstance(blob, dict): + for field in MCP_CREDENTIAL_SECRET_FIELDS: + value = blob.get(field) + if isinstance(value, str): + yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) + return + if secret_column.table == "LiteLLM_MCPServerTable" and secret_column.column == "env_vars": + # Only global-scope entries carry an encrypted value; user-scope placeholders + # hold a name and scope in plaintext by construction. + from litellm.proxy._experimental.mcp_server.db import is_global_env_var_scope + + entries: Final[JsonValue] = stored + if isinstance(entries, list): + for entry in entries: + if isinstance(entry, dict) and is_global_env_var_scope(entry.get("scope")): + value = entry.get("value") + if isinstance(value, str): + yield value.removeprefix(CALLBACK_VAR_ENCRYPTED_PREFIX) + return yield from _string_leaves(stored) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ca92d00c9c1..20d1bc1405b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1084,6 +1084,11 @@ def get_dynamic_logging_metadata( team_metadata: Final = decrypt_callback_vars(user_api_key_dict.team_metadata) callback_settings: Final = team_metadata.get("callback_settings", None) or {} callback_settings_obj = TeamCallbackMetadata(**callback_settings) + # Drop vars that still carry the marker so an unreadable ciphertext is never + # handed onward as the live credential; the variable reads as unset. + callback_settings_obj.callback_vars = cast( # cast-ok: filtering keeps the model's str values + "dict[str, str]", _without_unreadable_callback_vars(callback_settings_obj.callback_vars or {}) + ) verbose_proxy_logger.debug("Team callback settings activated: %s", callback_settings_obj) ######################################################################################### # Enter here when configured on the config.yaml file. @@ -1200,6 +1205,30 @@ def resolve_tenant_otel_destinations( ) +def _without_unreadable_callback_vars( + callback_vars: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: a fresh plain mapping the callers merge into settings + """The vars without values that still carry the encrypted-value marker. + + A marked value this install cannot decrypt (wrong key, or legacy ciphertext without + PyNaCl) is never a usable credential: handing it onward — into request kwargs or OTEL + headers — would treat the ciphertext blob as the live secret. The variable reads as + unset instead; the stored value itself is untouched. + """ + usable: Final[dict[str, object]] = { # mutable-ok: filtered copy handed to the caller + var: value + for var, value in callback_vars.items() + if not (isinstance(value, str) and value.startswith(CALLBACK_VAR_ENCRYPTED_PREFIX)) + } + if len(usable) != len(callback_vars): + verbose_proxy_logger.warning( + "Ignoring %d callback variable(s) still holding unreadable encrypted values: %s", + len(callback_vars) - len(usable), + sorted(set(callback_vars) - set(usable)), + ) + return usable + + def _tenant_destination( name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None ) -> "OtelDestination | None": @@ -1207,10 +1236,15 @@ def _tenant_destination( from litellm.integrations.otel.presets.destinations import destination_for callback_vars: Final = MappingProxyType( - dict( - itertools.chain.from_iterable( - callback.callback_vars.items() for callback in callbacks if callback.callback_name == name - ) + cast( # cast-ok: filtering AddTeamCallback.callback_vars keeps its dict[str, str] + "dict[str, str]", + _without_unreadable_callback_vars( + dict( + itertools.chain.from_iterable( + callback.callback_vars.items() for callback in callbacks if callback.callback_name == name + ) + ) + ), ) ) return destination_for( diff --git a/tests/unit/proxy/common_utils/test_callback_utils.py b/tests/unit/proxy/common_utils/test_callback_utils.py index 8d70f9504e3..faade805b3c 100644 --- a/tests/unit/proxy/common_utils/test_callback_utils.py +++ b/tests/unit/proxy/common_utils/test_callback_utils.py @@ -3,7 +3,7 @@ import json import sys from types import ModuleType, SimpleNamespace from typing import Final -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest @@ -780,3 +780,29 @@ def test_decrypt_callback_vars_round_trips_a_marked_value_under_the_current_key( decrypted = decrypt_callback_vars({"callback_settings": {"callback_vars": {"langfuse_secret_key": marked}}}) assert decrypted["callback_settings"]["callback_vars"]["langfuse_secret_key"] == "the-secret" + + +def test_deprecated_callback_settings_branch_drops_a_marked_value(monkeypatch): + """The deprecated metadata['callback_settings'] shape must not hand a still-marked + (unreadable) ciphertext to a provider as the live credential either.""" + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import get_dynamic_logging_metadata + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-callback-decrypt-salt") + marked = "litellm_enc::" + str(encrypt_value_helper("secret", new_encryption_key="sk-another-salt")) + team_metadata: Final = { + "callback_settings": { + "success_callback": ["langfuse"], + "callback_vars": {"langfuse_secret_key": marked, "langfuse_environment": "production"}, + } + } + auth: Final = UserAPIKeyAuth( + api_key="sk-test", + team_id="team-1", + user_role="proxy_admin", + team_metadata=team_metadata, + ) + + resolved = get_dynamic_logging_metadata(user_api_key_dict=auth, proxy_config=MagicMock()) + + assert resolved.callback_vars == {"langfuse_environment": "production"} diff --git a/tests/unit/proxy/db/test_master_key_migration.py b/tests/unit/proxy/db/test_master_key_migration.py index b06623f8c1a..5ad0e2ca993 100644 --- a/tests/unit/proxy/db/test_master_key_migration.py +++ b/tests/unit/proxy/db/test_master_key_migration.py @@ -772,3 +772,44 @@ async def test_integration_settings_config_rows_count_their_encrypted_fields(mon assert decrypt_if_encrypted_with(tables["LiteLLM_Config"][0]["param_value"]["api_key"], NEW_KEY) == "vantage-key" assert tables["LiteLLM_Config"][0]["param_value"]["base_url"] == "https://api.vantage.sh" assert tables["LiteLLM_Config"][1]["param_value"]["timezone"] == "utc" + + +@pytest.mark.asyncio +async def test_mcp_metadata_and_user_scope_env_vars_do_not_block_a_pynacl_free_migration(monkeypatch): + """MCP credentials hold plaintext metadata (region, scopes) next to the secret fields, + and env_vars hold plaintext user-scope placeholders: only writer-encrypted leaves + count, so a no-PyNaCl install still converges.""" + import sys + + monkeypatch.setitem(sys.modules, "nacl", None) + monkeypatch.setitem(sys.modules, "nacl.secret", None) + tables: Tables = { + "LiteLLM_MCPServerTable": [ + { + "server_id": "mcp-1", + "credentials": { + "auth_value": _encrypted("mcp-secret"), + "aws_region_name": "us-east-1", + "scopes": ["read"], + }, + "env_vars": [{"name": "API_TOKEN", "scope": "user"}], + } + ], + } + database = _FakeDatabase(tables) + + assert await count_values_encrypted_with(database, PREVIOUS_KEY) == 1 # only auth_value + + outcome = await migrate_from_previous_master_key( + previous_master_key=PREVIOUS_KEY, + master_key=NEW_KEY, + salt_key_is_set=False, + database=database, + log=lambda message: None, + ) + + assert outcome == Migrated(migrated=1, remaining=0, unreadable=0) + credentials = tables["LiteLLM_MCPServerTable"][0]["credentials"] + assert credentials["aws_region_name"] == "us-east-1" + assert credentials["scopes"] == ["read"] + assert tables["LiteLLM_MCPServerTable"][0]["env_vars"] == [{"name": "API_TOKEN", "scope": "user"}]