test(e2e): master key rotation re-encrypts team + key callback_vars

tests/e2e/coverage_registry/other.yaml

@@ -26,3 +26,4 @@
 - {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"}
 - {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"}
 - {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"}
+- {id: other.key_mgmt.master_rotation.reencrypts_callback_vars, module: other, tier: P1, area: auth, assertions: [reencrypts_callback_vars], source: "common_utils/callback_utils.py rotate_callback_vars_master_key", rationale: "Master key rotation re-encrypts team + key callback_vars under the new key so they don't strand under the old key", fail_before_fix: proven}

tests/e2e/management/management_client.py

@@ -17,6 +17,8 @@ from models import (
     KeyGenerateBody,
     KeyListParams,
     KeyListResponse,
+    KeyRegenerateBody,
+    KeyRegenerateResponse,
     KeyUpdateBody,
     OrgDeleteBody,
     OrgInfoParams,
@@ -74,6 +76,20 @@ class ManagementClient:
             )
         )
 
+    def rotate_master_key(self, current_master_key: str, new_master_key: str) -> None:
+        """Rotate the proxy master key via /key/regenerate, re-encrypting every
+        at-rest secret (including team/key callback_vars) under new_master_key.
+        Authenticated with the current master key, which stays valid in-memory
+        until the proxy is restarted with the new value."""
+        _ = unwrap(
+            self.gateway.transport.post(
+                "/key/regenerate",
+                headers=self.gateway.transport.bearer(current_master_key),
+                json=KeyRegenerateBody(key=current_master_key, new_master_key=new_master_key),
+                response_type=KeyRegenerateResponse,
+            )
+        )
+
     def key_alias_count(self, key_alias: str) -> int:
         return unwrap(
             self.gateway.transport.get(

tests/e2e/management/test_master_key_rotation_e2e.py

@@ -0,0 +1,121 @@
+"""Live e2e: master key rotation re-encrypts team and key callback_vars.
+
+Logging callback credentials (Langfuse / Langsmith secrets) live encrypted at
+rest in LiteLLM_TeamTable.metadata and LiteLLM_VerificationToken.metadata. Master
+key rotation used to re-encrypt models, config env vars, MCP credentials, and the
+credentials table but skipped these callback_vars, so after a rotation they stayed
+encrypted under the old key and produced recurring decryption errors.
+
+This drives the real /key/regenerate rotation against a live proxy: it creates a
+team and a key carrying encrypted callback_vars, rotates the master key to a fresh
+value, and asserts the stored ciphertext for both rows was re-encrypted (changed)
+while staying encrypted at rest and never leaking the plaintext secret. The
+non-sensitive langfuse_host is left untouched. Reverting the fix leaves the
+ciphertext byte-for-byte identical after rotation, so the change assertion fails.
+
+The rotation re-encrypts every at-rest secret under the new key while the running
+proxy keeps the old key in memory (a restart with the new key is the operator's
+next step), so the teardown rotates the master key back to the suite's key to
+leave the shared stack decryptable for other tests.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_config import MASTER_KEY, unique_marker
+from lifecycle import ResourceManager
+from management_client import ManagementClient
+from models import (
+    CallbackMetadata,
+    CallbackVars,
+    KeyGenerateBody,
+    LoggingCallbackEntry,
+    TeamNewBody,
+)
+
+pytestmark = pytest.mark.e2e
+
+ENCRYPTED_PREFIX = "litellm_enc::"
+LANGFUSE_HOST = "https://cloud.langfuse.com"
+
+
+def _callback_metadata(marker: str, secret: str) -> CallbackMetadata:
+    return CallbackMetadata(
+        logging=[
+            LoggingCallbackEntry(
+                callback_name="langfuse",
+                callback_type="success",
+                callback_vars=CallbackVars(
+                    langfuse_public_key=f"pk-lf-{marker}",
+                    langfuse_secret_key=secret,
+                    langfuse_host=LANGFUSE_HOST,
+                ),
+            )
+        ]
+    )
+
+
+def _callback_vars(metadata: CallbackMetadata | None) -> CallbackVars:
+    assert metadata is not None and metadata.logging, (
+        f"expected callback logging metadata to be persisted, got {metadata!r}"
+    )
+    return metadata.logging[0].callback_vars
+
+
+class TestMasterKeyRotationCallbackVars:
+    @pytest.mark.covers("other.key_mgmt.master_rotation.reencrypts_callback_vars")
+    def test_rotation_reencrypts_team_and_key_callback_vars(
+        self, client: ManagementClient, resources: ResourceManager
+    ) -> None:
+        marker = unique_marker()
+        team_secret = f"sk-lf-team-secret-{marker}"
+        key_secret = f"sk-lf-key-secret-{marker}"
+
+        team_id = client.create_team(
+            TeamNewBody(team_alias=f"e2e-rot-team-{marker}", metadata=_callback_metadata(marker, team_secret))
+        )
+        resources.defer(lambda: client.delete_team(team_id))
+        key = client.gateway.generate_key(
+            KeyGenerateBody(key_alias=f"e2e-rot-key-{marker}", metadata=_callback_metadata(marker, key_secret))
+        )
+        resources.defer(lambda: client.gateway.delete_key(key))
+
+        team_before = _callback_vars(client.team_info(team_id).metadata)
+        key_before = _callback_vars(client.gateway.key_info(key).metadata)
+
+        for label, plaintext, ciphertext in (
+            ("team", team_secret, team_before.langfuse_secret_key),
+            ("key", key_secret, key_before.langfuse_secret_key),
+        ):
+            assert ciphertext is not None and ciphertext.startswith(ENCRYPTED_PREFIX), (
+                f"{label} langfuse_secret_key must be encrypted at rest, got {ciphertext!r}"
+            )
+            assert plaintext not in ciphertext, f"{label} secret leaked as plaintext at rest: {ciphertext!r}"
+
+        new_master_key = f"sk-rotated-{marker}"
+        client.rotate_master_key(MASTER_KEY, new_master_key)
+        resources.defer(lambda: client.rotate_master_key(MASTER_KEY, MASTER_KEY))
+
+        team_after = _callback_vars(client.team_info(team_id).metadata)
+        key_after = _callback_vars(client.gateway.key_info(key).metadata)
+
+        for label, plaintext, before, after in (
+            ("team", team_secret, team_before.langfuse_secret_key, team_after.langfuse_secret_key),
+            ("key", key_secret, key_before.langfuse_secret_key, key_after.langfuse_secret_key),
+        ):
+            assert after is not None and after.startswith(ENCRYPTED_PREFIX), (
+                f"{label} langfuse_secret_key must stay encrypted after rotation, got {after!r}"
+            )
+            assert after != before, (
+                f"{label} langfuse_secret_key was not re-encrypted on master key rotation; "
+                f"it is still the old-key ciphertext {after!r}"
+            )
+            assert plaintext not in after, f"{label} secret leaked as plaintext after rotation: {after!r}"
+
+        assert team_after.langfuse_host == LANGFUSE_HOST, (
+            f"non-sensitive langfuse_host must be untouched by rotation, got {team_after.langfuse_host!r}"
+        )
+        assert key_after.langfuse_host == LANGFUSE_HOST, (
+            f"non-sensitive langfuse_host must be untouched by rotation, got {key_after.langfuse_host!r}"
+        )

tests/e2e/models.py

@@ -18,6 +18,22 @@ class ModelBudgetEntry(BaseModel):
     time_period: str
 
 
+class CallbackVars(BaseModel):
+    langfuse_public_key: str | None = None
+    langfuse_secret_key: str | None = None
+    langfuse_host: str | None = None
+
+
+class LoggingCallbackEntry(BaseModel):
+    callback_name: str
+    callback_type: str
+    callback_vars: CallbackVars
+
+
+class CallbackMetadata(BaseModel):
+    logging: list[LoggingCallbackEntry] = []
+
+
 class BudgetWindow(BaseModel):
     budget_duration: str
     max_budget: float
@@ -39,12 +55,22 @@ class KeyGenerateBody(BaseModel):
     tpm_limit: int | None = None
     rpm_limit: int | None = None
     allowed_routes: list[str] | None = None
+    metadata: CallbackMetadata | None = None
 
 
 class KeyGenerateResponse(BaseModel):
     key: str
 
 
+class KeyRegenerateBody(BaseModel):
+    key: str
+    new_master_key: str
+
+
+class KeyRegenerateResponse(BaseModel):
+    key: str
+
+
 class KeyDeleteBody(BaseModel):
     keys: list[str]
 
@@ -70,6 +96,7 @@ class KeyInfo(BaseModel):
     budget_reset_at: str | None = None
     budget_id: str | None = None
     litellm_budget_table: LiteLLMBudgetTable | None = None
+    metadata: CallbackMetadata | None = None
 
 
 class KeyInfoResponse(BaseModel):
@@ -439,6 +466,7 @@ class TeamNewBody(BaseModel):
     team_alias: str
     models: list[str] = []
     team_id: str | None = None
+    metadata: CallbackMetadata | None = None
 
 
 class TeamNewResponse(BaseModel):
@@ -453,6 +481,7 @@ class TeamData(BaseModel):
     team_alias: str | None = None
     models: list[str] = []
     members_with_roles: list[TeamMemberEntry] = []
+    metadata: CallbackMetadata | None = None
 
 
 class TeamInfoResponse(BaseModel):