feat(usage-export): redact credential-named values inside exported key and user metadata

litellm/repositories/daily_activity_repository.py

@@ -1,5 +1,6 @@
 import asyncio
 from collections.abc import AsyncIterator, Mapping, Sequence
+from dataclasses import replace
 from datetime import datetime
 from itertools import groupby
 from types import MappingProxyType
@@ -11,6 +12,7 @@ from typing_extensions import assert_never
 from litellm import constants
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import UNKNOWN_MODEL_SPEND_LOG_MODEL
+from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
 from litellm.repositories.chunked_in import find_many_in
 from litellm.repositories.daily_activity_sql import (
     ExportCursor,
@@ -140,6 +142,16 @@ def _daily_rows_table(
     raise AssertionError("unreachable")
 
 
+def _with_redacted_metadata(row: ExportRow) -> ExportRow:
+    if not row.key_metadata and not row.user_metadata:
+        return row
+    return replace(
+        row,
+        key_metadata=redact_credentials_in_payload(row.key_metadata),
+        user_metadata=redact_credentials_in_payload(row.user_metadata),
+    )
+
+
 def _next_export_cursor(batch: tuple[ExportRow, ...], export_type: ExportType) -> ExportCursor:
     last: Final = batch[-1]
     cursor_key: Final = (
@@ -276,7 +288,7 @@ class DailyActivityRepository:
                 )
             )
             for row in batch:
-                yield row
+                yield _with_redacted_metadata(row)
             if len(batch) < batch_size:
                 return
             cursor = _next_export_cursor(batch, export_type)

tests/e2e/coverage_registry/quota_management.yaml

@@ -40,7 +40,7 @@
 - {id: quota_management.budget.fallback.routes_to_fallback, module: quota_management, tier: P1, behavior: budget, variant: fallback, assertions: [routes_to_fallback], exercised_on: [messages], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "budget_fallbacks reroute to the fallback model once the primary's budget is exhausted"}
 - {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"}
 - {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"}
-- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata, empty_metadata_without_admin_view], exercised_on: [chat_completions], source: "repositories/daily_activity_sql.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects for callers with admin view and {} for everyone else"}
+- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata, empty_metadata_without_admin_view, redacts_credential_named_values], exercised_on: [chat_completions], source: "repositories/daily_activity_repository.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects for callers with admin view with credential-named values replaced by REDACTED, and {} for everyone else"}
 - {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"}
 - {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"}
 - {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"}

tests/e2e/models.py

@@ -59,6 +59,7 @@ class KeyMetadata(BaseModel):
     batch_enqueued_token_limit: int | None = None
     tag: str | None = None
     guardrails: list[str] | None = None
+    webhook_token: str | None = None
 
 
 class ObjectPermission(BaseModel):

tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py

@@ -46,10 +46,12 @@ class UsageExportParams(BaseModel):
 
 class UsageExportKeyMetadata(BaseModel):
     tag: str | None = None
+    webhook_token: str | None = None
 
 
 class UsageExportUserMetadata(BaseModel):
     department: str | None = None
+    ldap_password: str | None = None
 
 
 class UsageExportRow(BaseModel):
@@ -99,7 +101,7 @@ class TestUsageExportMetadata:
                     user_email=f"e2e-usage-export-{marker}@example.com",
                     user_role="internal_user",
                     user_id=f"e2e-usage-export-{marker}",
-                    metadata={"department": f"research-{marker}"},
+                    metadata={"department": f"research-{marker}", "ldap_password": f"pw-{marker}"},
                 ),
                 response_type=UserNewResponse,
             )
@@ -110,7 +112,7 @@ class TestUsageExportMetadata:
                 user_id=user_id,
                 models=[model],
                 key_alias=f"e2e-usage-export-{marker}",
-                metadata=KeyMetadata(tag=f"cost-center-{marker}"),
+                metadata=KeyMetadata(tag=f"cost-center-{marker}", webhook_token=f"whsec-{marker}"),
             )
         )
         resources.defer(lambda: client.proxy.delete_key(record.key))
@@ -152,6 +154,9 @@ class TestUsageExportMetadata:
             assert row.user_metadata.department == f"research-{marker}", (
                 f"user metadata not exported: {row.user_metadata}"
             )
+            assert (row.key_metadata.webhook_token, row.user_metadata.ldap_password) == ("REDACTED", "REDACTED"), (
+                f"credential-named metadata values reached the export unmasked: {row}"
+            )
 
         own_view: Final = client.proxy.transport.probe(
             EXPORT_ROUTE,

tests/integration/spend/_daily_activity_fixtures.py

@@ -176,7 +176,7 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str,
                     "alias-a",
                     "team-1",
                     "user-1",
-                    '{"tags": ["blue", "gold"], "cost_center": "cc-1", '
+                    '{"tags": ["blue", "gold"], "cost_center": "cc-1", "webhook_token": "whsec_1", '
                     '"logging": [{"callback_name": "langfuse", "callback_vars": {"langfuse_secret_key": "sk"}}], '
                     '"callback_settings": {"otel": {"headers": "x"}}, "secret_manager_settings": {"vault": "v"}}',
                     [],
@@ -214,7 +214,7 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str,
                 "user-1",
                 "user@example.com",
                 [],
-                '{"department": "research", "scim_enterprise": {"employeeNumber": "E-1"}, '
+                '{"department": "research", "ldap_password": "pw", "scim_enterprise": {"employeeNumber": "E-1"}, '
                 '"scim_entitlements": ["ent"], "scim_roles": ["role"]}',
             ),
         )

tests/integration/spend/test_daily_activity_repository.py

@@ -242,10 +242,14 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch:
             "user@example.com",
         )
         assert deleted_key_export.key_metadata == {"tags": ["archived"]}
-        assert deleted_key_export.user_metadata == {"department": "research"}
+        assert deleted_key_export.user_metadata == {"department": "research", "ldap_password": "REDACTED"}
         active_key_export: Final = next(row for row in exports if row.api_key == "key-a")
-        assert active_key_export.key_metadata == {"tags": ["blue", "gold"], "cost_center": "cc-1"}
-        assert active_key_export.user_metadata == {"department": "research"}
+        assert active_key_export.key_metadata == {
+            "tags": ["blue", "gold"],
+            "cost_center": "cc-1",
+            "webhook_token": "REDACTED",
+        }
+        assert active_key_export.user_metadata == {"department": "research", "ldap_password": "REDACTED"}
         unprivileged_exports: Final = tuple(
             [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY_WITH_KEYS)]
         )
@@ -265,7 +269,7 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch:
             "user@example.com",
             273.0,
         )
-        assert user_exports[0].user_metadata == {"department": "research"}
+        assert user_exports[0].user_metadata == {"department": "research", "ldap_password": "REDACTED"}
         assert user_exports[0].key_metadata == {}
         daily_export: Final = tuple(
             [

tests/unit/repositories/test_daily_activity_repository.py

@@ -678,6 +678,39 @@ async def test_daily_rows_exclusion_without_entity_filter_keeps_null_entity_rows
     assert database.litellm_dailyteamspend.find_many_calls == [expected_where]
 
 
+@pytest.mark.asyncio
+async def test_export_redacts_credential_named_values_inside_metadata_objects() -> None:
+    database = _FakeDatabase(
+        (
+            (
+                {
+                    **_export_row("key-1"),
+                    "key_metadata": {
+                        "cost_center": "cc-1",
+                        "api_key": "sk-live-1234567890",
+                        "webhook": {"url": "https://hooks.example", "auth_token": "t0k3n"},
+                        "owners": [{"name": "ops", "password": "pw"}],
+                    },
+                    "user_metadata": {"department": "research", "github_token": "ghp_secret"},
+                },
+                {**_export_row("key-2"), "key_metadata": {}, "user_metadata": {}},
+            ),
+        )
+    )
+    repository, _ = _repository(database)
+
+    rows = [row async for row in repository.export_rows(_scope(), export_type=ExportType.DAILY_WITH_KEYS)]
+
+    assert rows[0].key_metadata == {
+        "cost_center": "cc-1",
+        "api_key": "REDACTED",
+        "webhook": {"url": "https://hooks.example", "auth_token": "REDACTED"},
+        "owners": [{"name": "ops", "password": "REDACTED"}],
+    }
+    assert rows[0].user_metadata == {"department": "research", "github_token": "REDACTED"}
+    assert (rows[1].key_metadata, rows[1].user_metadata) == ({}, {})
+
+
 @pytest.mark.asyncio
 async def test_export_is_lazy_and_uses_the_last_row_as_the_next_cursor(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(constants, "USAGE_EXPORT_BATCH_SIZE", 2)