From e964397a2243b80a8ba1654efc02158f82bfd4a4 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 3 Nov 2026 19:00:23 +0000 Subject: [PATCH] feat(usage-export): redact credential-named values inside exported key and user metadata --- .../repositories/daily_activity_repository.py | 14 +++++++- .../coverage_registry/quota_management.yaml | 2 +- tests/e2e/models.py | 1 + .../test_usage_export_metadata_e2e.py | 9 +++-- .../spend/_daily_activity_fixtures.py | 4 +-- .../spend/test_daily_activity_repository.py | 12 ++++--- .../test_daily_activity_repository.py | 33 +++++++++++++++++++ 7 files changed, 65 insertions(+), 10 deletions(-) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index e60e3cf2a8e..0527def01a5 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -1,5 +1,6 @@ import asyncio from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import replace from datetime import datetime from itertools import groupby from types import MappingProxyType @@ -11,6 +12,7 @@ from typing_extensions import assert_never from litellm import constants from litellm._logging import verbose_proxy_logger from litellm.constants import UNKNOWN_MODEL_SPEND_LOG_MODEL +from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.repositories.chunked_in import find_many_in from litellm.repositories.daily_activity_sql import ( ExportCursor, @@ -140,6 +142,16 @@ def _daily_rows_table( raise AssertionError("unreachable") +def _with_redacted_metadata(row: ExportRow) -> ExportRow: + if not row.key_metadata and not row.user_metadata: + return row + return replace( + row, + key_metadata=redact_credentials_in_payload(row.key_metadata), + user_metadata=redact_credentials_in_payload(row.user_metadata), + ) + + def _next_export_cursor(batch: tuple[ExportRow, ...], export_type: ExportType) -> ExportCursor: last: Final = batch[-1] cursor_key: Final = ( @@ -276,7 +288,7 @@ class DailyActivityRepository: ) ) for row in batch: - yield row + yield _with_redacted_metadata(row) if len(batch) < batch_size: return cursor = _next_export_cursor(batch, export_type) diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 56f5e7a66ca..0659de6febe 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -40,7 +40,7 @@ - {id: quota_management.budget.fallback.routes_to_fallback, module: quota_management, tier: P1, behavior: budget, variant: fallback, assertions: [routes_to_fallback], exercised_on: [messages], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "budget_fallbacks reroute to the fallback model once the primary's budget is exhausted"} - {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"} - {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"} -- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata, empty_metadata_without_admin_view], exercised_on: [chat_completions], source: "repositories/daily_activity_sql.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects for callers with admin view and {} for everyone else"} +- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata, empty_metadata_without_admin_view, redacts_credential_named_values], exercised_on: [chat_completions], source: "repositories/daily_activity_repository.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects for callers with admin view with credential-named values replaced by REDACTED, and {} for everyone else"} - {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"} - {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} - {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 679195276fc..7467d2fd085 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -59,6 +59,7 @@ class KeyMetadata(BaseModel): batch_enqueued_token_limit: int | None = None tag: str | None = None guardrails: list[str] | None = None + webhook_token: str | None = None class ObjectPermission(BaseModel): diff --git a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py index fee0c97d623..a1f5633f17e 100644 --- a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py @@ -46,10 +46,12 @@ class UsageExportParams(BaseModel): class UsageExportKeyMetadata(BaseModel): tag: str | None = None + webhook_token: str | None = None class UsageExportUserMetadata(BaseModel): department: str | None = None + ldap_password: str | None = None class UsageExportRow(BaseModel): @@ -99,7 +101,7 @@ class TestUsageExportMetadata: user_email=f"e2e-usage-export-{marker}@example.com", user_role="internal_user", user_id=f"e2e-usage-export-{marker}", - metadata={"department": f"research-{marker}"}, + metadata={"department": f"research-{marker}", "ldap_password": f"pw-{marker}"}, ), response_type=UserNewResponse, ) @@ -110,7 +112,7 @@ class TestUsageExportMetadata: user_id=user_id, models=[model], key_alias=f"e2e-usage-export-{marker}", - metadata=KeyMetadata(tag=f"cost-center-{marker}"), + metadata=KeyMetadata(tag=f"cost-center-{marker}", webhook_token=f"whsec-{marker}"), ) ) resources.defer(lambda: client.proxy.delete_key(record.key)) @@ -152,6 +154,9 @@ class TestUsageExportMetadata: assert row.user_metadata.department == f"research-{marker}", ( f"user metadata not exported: {row.user_metadata}" ) + assert (row.key_metadata.webhook_token, row.user_metadata.ldap_password) == ("REDACTED", "REDACTED"), ( + f"credential-named metadata values reached the export unmasked: {row}" + ) own_view: Final = client.proxy.transport.probe( EXPORT_ROUTE, diff --git a/tests/integration/spend/_daily_activity_fixtures.py b/tests/integration/spend/_daily_activity_fixtures.py index a7e4c0df918..85c434c6877 100644 --- a/tests/integration/spend/_daily_activity_fixtures.py +++ b/tests/integration/spend/_daily_activity_fixtures.py @@ -176,7 +176,7 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, "alias-a", "team-1", "user-1", - '{"tags": ["blue", "gold"], "cost_center": "cc-1", ' + '{"tags": ["blue", "gold"], "cost_center": "cc-1", "webhook_token": "whsec_1", ' '"logging": [{"callback_name": "langfuse", "callback_vars": {"langfuse_secret_key": "sk"}}], ' '"callback_settings": {"otel": {"headers": "x"}}, "secret_manager_settings": {"vault": "v"}}', [], @@ -214,7 +214,7 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, "user-1", "user@example.com", [], - '{"department": "research", "scim_enterprise": {"employeeNumber": "E-1"}, ' + '{"department": "research", "ldap_password": "pw", "scim_enterprise": {"employeeNumber": "E-1"}, ' '"scim_entitlements": ["ent"], "scim_roles": ["role"]}', ), ) diff --git a/tests/integration/spend/test_daily_activity_repository.py b/tests/integration/spend/test_daily_activity_repository.py index 6229bbc13be..3df2b934112 100644 --- a/tests/integration/spend/test_daily_activity_repository.py +++ b/tests/integration/spend/test_daily_activity_repository.py @@ -242,10 +242,14 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: "user@example.com", ) assert deleted_key_export.key_metadata == {"tags": ["archived"]} - assert deleted_key_export.user_metadata == {"department": "research"} + assert deleted_key_export.user_metadata == {"department": "research", "ldap_password": "REDACTED"} active_key_export: Final = next(row for row in exports if row.api_key == "key-a") - assert active_key_export.key_metadata == {"tags": ["blue", "gold"], "cost_center": "cc-1"} - assert active_key_export.user_metadata == {"department": "research"} + assert active_key_export.key_metadata == { + "tags": ["blue", "gold"], + "cost_center": "cc-1", + "webhook_token": "REDACTED", + } + assert active_key_export.user_metadata == {"department": "research", "ldap_password": "REDACTED"} unprivileged_exports: Final = tuple( [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY_WITH_KEYS)] ) @@ -265,7 +269,7 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: "user@example.com", 273.0, ) - assert user_exports[0].user_metadata == {"department": "research"} + assert user_exports[0].user_metadata == {"department": "research", "ldap_password": "REDACTED"} assert user_exports[0].key_metadata == {} daily_export: Final = tuple( [ diff --git a/tests/unit/repositories/test_daily_activity_repository.py b/tests/unit/repositories/test_daily_activity_repository.py index d93a75476a2..ddcc0f0ff11 100644 --- a/tests/unit/repositories/test_daily_activity_repository.py +++ b/tests/unit/repositories/test_daily_activity_repository.py @@ -678,6 +678,39 @@ async def test_daily_rows_exclusion_without_entity_filter_keeps_null_entity_rows assert database.litellm_dailyteamspend.find_many_calls == [expected_where] +@pytest.mark.asyncio +async def test_export_redacts_credential_named_values_inside_metadata_objects() -> None: + database = _FakeDatabase( + ( + ( + { + **_export_row("key-1"), + "key_metadata": { + "cost_center": "cc-1", + "api_key": "sk-live-1234567890", + "webhook": {"url": "https://hooks.example", "auth_token": "t0k3n"}, + "owners": [{"name": "ops", "password": "pw"}], + }, + "user_metadata": {"department": "research", "github_token": "ghp_secret"}, + }, + {**_export_row("key-2"), "key_metadata": {}, "user_metadata": {}}, + ), + ) + ) + repository, _ = _repository(database) + + rows = [row async for row in repository.export_rows(_scope(), export_type=ExportType.DAILY_WITH_KEYS)] + + assert rows[0].key_metadata == { + "cost_center": "cc-1", + "api_key": "REDACTED", + "webhook": {"url": "https://hooks.example", "auth_token": "REDACTED"}, + "owners": [{"name": "ops", "password": "REDACTED"}], + } + assert rows[0].user_metadata == {"department": "research", "github_token": "REDACTED"} + assert (rows[1].key_metadata, rows[1].user_metadata) == ({}, {}) + + @pytest.mark.asyncio async def test_export_is_lazy_and_uses_the_last_row_as_the_next_cursor(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(constants, "USAGE_EXPORT_BATCH_SIZE", 2)