diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 26988609140..2352524b514 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -718,7 +718,15 @@ async def _rotate_callback_vars_table( metadata = getattr(row, "metadata", None) if not _has_encrypted_callback_vars(metadata): continue - re_encrypted = encrypt_callback_vars(decrypt_callback_vars(metadata), new_encryption_key=new_master_key) + decrypted = decrypt_callback_vars(metadata) + if _has_encrypted_callback_vars(decrypted): + verbose_proxy_logger.warning( + "rotate_callback_vars_master_key: %s %s has callback_vars that failed to decrypt under the " + "current key; those values are left encrypted under the old key", + table_name, + getattr(row, pk), + ) + re_encrypted = encrypt_callback_vars(decrypted, new_encryption_key=new_master_key) await table.update( where={pk: getattr(row, pk)}, data={"metadata": json.dumps(re_encrypted)}, diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 3547daa859a..aa5f566e0c4 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -4266,7 +4266,7 @@ async def _rotate_master_key( prisma_client=prisma_client, new_master_key=new_master_key, ) - except prisma.errors.PrismaError as e: + except Exception as e: # noqa: BLE001 # log-and-continue so a callback_vars failure never aborts a full master key rotation, matching the credential step above verbose_proxy_logger.warning("Failed to rotate callback_vars: %s", str(e)) diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index fd0210a2e4c..d7a835528b2 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -502,3 +502,42 @@ async def test_rotate_callback_vars_master_key_reencrypts_under_new_key(monkeypa recovered["callback_settings"]["callback_vars"]["langsmith_api_key"] == "ls-api-key" ) + + +@pytest.mark.asyncio +async def test_rotate_callback_vars_master_key_warns_on_undecryptable_value(monkeypatch): + """A value that fails to decrypt under the current key stays under the old key + instead of being silently reported as rotated, so rotation logs a warning. + """ + old_key = "old-key-aaaaaaaaaaaaaaaaaaaaaaaa" + new_key = "new-key-bbbbbbbbbbbbbbbbbbbbbbbb" + foreign_key = "foreign-key-cccccccccccccccccccccccc" + + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setenv("LITELLM_SALT_KEY", old_key) + + team_meta = encrypt_callback_vars(_sample_metadata(), new_encryption_key=foreign_key) + team_row = SimpleNamespace(team_id="team-1", metadata=team_meta) + + client = MagicMock() + client.db.litellm_teamtable.find_many = AsyncMock(return_value=[team_row]) + client.db.litellm_teamtable.update = AsyncMock() + client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + client.db.litellm_verificationtoken.update = AsyncMock() + + warn = MagicMock() + monkeypatch.setattr( + "litellm.proxy.common_utils.callback_utils.verbose_proxy_logger.warning", warn + ) + + await rotate_callback_vars_master_key(client, new_master_key=new_key) + + warn.assert_called_once() + assert "team-1" in warn.call_args.args + + written = json.loads( + client.db.litellm_teamtable.update.call_args.kwargs["data"]["metadata"] + ) + monkeypatch.setenv("LITELLM_SALT_KEY", new_key) + still_stuck = decrypt_callback_vars(written)["logging"][0]["callback_vars"] + assert still_stuck["langfuse_secret_key"].startswith("litellm_enc::")