From f26209f560336cd152eecd9bcef114023fcfcf83 Mon Sep 17 00:00:00 2001 From: jesus Date: Sat, 31 Oct 2026 00:08:39 +0000 Subject: [PATCH 01/11] feat(usage-export): include key and user metadata objects in daily activity export rows --- .../daily_activity_routes.py | 2 + litellm/repositories/daily_activity_sql.py | 26 ++- litellm/types/repositories/daily_activity.py | 2 + .../coverage_registry/quota_management.yaml | 1 + tests/e2e/models.py | 1 + .../test_usage_export_metadata_e2e.py | 152 ++++++++++++++++++ .../spend/_daily_activity_fixtures.py | 6 +- .../spend/test_daily_activity_repository.py | 12 +- .../test_daily_activity_routes.py | 8 + .../repositories/test_daily_activity_sql.py | 28 ++++ 10 files changed, 227 insertions(+), 11 deletions(-) create mode 100644 tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 1a3445aae52..1abda1aae33 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -221,6 +221,8 @@ def _search_rows(keys: Sequence[str], response: SpendAnalyticsPaginatedResponse) def _csv_cell(value: object) -> object: + if isinstance(value, Mapping): + return json.dumps(value, separators=(",", ":")) if isinstance(value, str) and value.startswith(("=", "+", "-", "@", "\t", "\r")): return f"'{value}" return value diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 4bb52767de6..5393fc42100 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -520,12 +520,23 @@ def build_export_sql( ) +_EMPTY_METADATA_FIELDS: Final = ", '{}'::jsonb AS key_metadata, '{}'::jsonb AS user_metadata" +_USER_METADATA_FIELD: Final = ( + "COALESCE(MIN(CASE WHEN jsonb_typeof(u.metadata) = 'object' THEN u.metadata::text END), '{}')::jsonb " + "AS user_metadata" +) +_KEY_METADATA_FIELD: Final = ( + "COALESCE(MIN(CASE WHEN jsonb_typeof(COALESCE(vt.metadata, dvt.metadata)) = 'object' " + "THEN COALESCE(vt.metadata, dvt.metadata)::text END), '{}')::jsonb AS key_metadata" +) + + def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str, ...]]: if export_type is ExportType.DAILY: return ( "''", "NULL::text", - "NULL::text AS key_alias, NULL::text AS user_id, NULL::text AS user_email", + "NULL::text AS key_alias, NULL::text AS user_id, NULL::text AS user_email" + _EMPTY_METADATA_FIELDS, (), ) if export_type is ExportType.DAILY_WITH_KEYS: @@ -534,11 +545,14 @@ def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str, _API_KEY_EXPR, "MAX(CASE WHEN scoped.api_key = '' THEN 'Unattributed' " "ELSE COALESCE(vt.key_alias, dvt.key_alias) END) AS key_alias, " - "MAX(COALESCE(vt.user_id, dvt.user_id)) AS user_id, MAX(u.user_email) AS user_email", + "MAX(COALESCE(vt.user_id, dvt.user_id)) AS user_id, MAX(u.user_email) AS user_email, " + + _KEY_METADATA_FIELD + + ", " + + _USER_METADATA_FIELD, ( "LEFT JOIN \"LiteLLM_VerificationToken\" vt ON vt.token = scoped.api_key AND scoped.api_key <> ''", """LEFT JOIN LATERAL ( - SELECT key_alias, user_id + SELECT key_alias, user_id, metadata FROM "LiteLLM_DeletedVerificationToken" WHERE token = scoped.api_key ORDER BY deleted_at DESC @@ -551,7 +565,7 @@ def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str, return ( _MODEL_EXPR, "NULL::text", - "NULL::text AS key_alias, NULL::text AS user_id, NULL::text AS user_email", + "NULL::text AS key_alias, NULL::text AS user_id, NULL::text AS user_email" + _EMPTY_METADATA_FIELDS, (), ) if export_type is ExportType.DAILY_WITH_USERS: @@ -559,11 +573,11 @@ def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str, "COALESCE(vt.user_id, dvt.user_id, '')", "NULL::text", "NULL::text AS key_alias, MAX(COALESCE(vt.user_id, dvt.user_id)) AS user_id, " - "MAX(u.user_email) AS user_email", + "MAX(u.user_email) AS user_email, '{}'::jsonb AS key_metadata, " + _USER_METADATA_FIELD, ( 'LEFT JOIN "LiteLLM_VerificationToken" vt ON vt.token = scoped.api_key', """LEFT JOIN LATERAL ( - SELECT key_alias, user_id + SELECT key_alias, user_id, metadata FROM "LiteLLM_DeletedVerificationToken" WHERE token = scoped.api_key ORDER BY deleted_at DESC diff --git a/litellm/types/repositories/daily_activity.py b/litellm/types/repositories/daily_activity.py index b013ce43113..7baf1600c03 100644 --- a/litellm/types/repositories/daily_activity.py +++ b/litellm/types/repositories/daily_activity.py @@ -126,6 +126,8 @@ class ExportRow: failed_requests: int cache_read_input_tokens: int cache_creation_input_tokens: int + key_metadata: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + user_metadata: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) @dataclass(frozen=True, slots=True) diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index cbaf46130d6..75dd883742f 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -40,6 +40,7 @@ - {id: quota_management.budget.fallback.routes_to_fallback, module: quota_management, tier: P1, behavior: budget, variant: fallback, assertions: [routes_to_fallback], exercised_on: [messages], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "budget_fallbacks reroute to the fallback model once the primary's budget is exhausted"} - {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"} - {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"} +- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata], exercised_on: [chat_completions], source: "repositories/daily_activity_sql.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects"} - {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"} - {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} - {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index f76e5cec8e1..4ae54de9df5 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1677,6 +1677,7 @@ class UserNewBody(BaseModel): user_id: str | None = None auto_create_key: bool | None = None max_budget: float | None = None + metadata: dict[str, str] | None = None class UserNewResponse(BaseModel): diff --git a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py new file mode 100644 index 00000000000..e70eba87689 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py @@ -0,0 +1,152 @@ +"""The usage export carries the stored key and user metadata objects. + +A customer charging usage back by cost center keeps that cost center in the +virtual key's metadata and the owner's user metadata. Before LIT-8524 the +Usage page export only carried the key alias, user id and email, so every +exported row needed a /key/info and /user/info round trip to recover the +metadata. The export now joins both objects onto each daily_with_keys row. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest +from e2e_config import provider_edge_base, unique_marker +from e2e_metadata import Domain, Mode, Provider, Subject, meta +from lifecycle import ResourceManager +from models import ( + ChatBody, + ChatMessage, + KeyGenerateBody, + KeyMetadata, + LiteLLMParamsBody, + UserNewBody, + UserNewResponse, +) +from pydantic import BaseModel, TypeAdapter +from spend_e2e_client import SpendClient, unwrap +from spend_reconciliation import BACKEND, INPUT_RATE, OUTPUT_RATE + +pytestmark = pytest.mark.e2e + +EXPORT_ROUTE: Final = "/user/daily/activity/export" + + +class UsageExportParams(BaseModel): + start_date: str + end_date: str + api_key: str + export_type: str = "daily_with_keys" + format: str = "json" + + +class UsageExportKeyMetadata(BaseModel): + tag: str | None = None + + +class UsageExportUserMetadata(BaseModel): + department: str | None = None + + +class UsageExportRow(BaseModel): + api_key: str | None = None + user_id: str | None = None + key_metadata: UsageExportKeyMetadata + user_metadata: UsageExportUserMetadata + + +_EXPORT_ROWS: Final = TypeAdapter(list[UsageExportRow]) + + +class TestUsageExportMetadata: + @pytest.mark.replayable + @pytest.mark.covers("quota_management.spend_tracking.usage_export.carries_key_and_user_metadata") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + providers=(Provider.OPENAI,), + models=(BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_daily_with_keys_export_row_carries_key_and_user_metadata( + self, client: SpendClient, resources: ResourceManager + ) -> None: + started: Final = datetime.now(timezone.utc) + marker: Final = unique_marker() + base: Final = provider_edge_base("openai") + model: Final = f"e2e-usage-export-{marker}" + model_id: Final = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=BACKEND, + api_key="os.environ/OPENAI_API_KEY", + api_base=None if base is None else f"{base}/v1", + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + user_id: Final = unwrap( + client.proxy.transport.post( + "/user/new", + headers=client.proxy.transport.master, + json=UserNewBody( + user_email=f"e2e-usage-export-{marker}@example.com", + user_role="internal_user", + user_id=f"e2e-usage-export-{marker}", + metadata={"department": f"research-{marker}"}, + ), + response_type=UserNewResponse, + ) + ).user_id + resources.defer(lambda: client.delete_user(user_id)) + record: Final = client.generate_key_record( + KeyGenerateBody( + user_id=user_id, + models=[model], + key_alias=f"e2e-usage-export-{marker}", + metadata=KeyMetadata(tag=f"cost-center-{marker}"), + ) + ) + resources.defer(lambda: client.proxy.delete_key(record.key)) + assert record.token, "/key/generate answered without the key's token hash" + token: Final = record.token + + response: Final = unwrap( + client.proxy.chat( + record.key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"Reply with one word. {marker}")], + max_completion_tokens=32, + ), + ) + ) + assert response.id, "successful response must have an ID" + end: Final = datetime.now(timezone.utc) + timedelta(days=1) + activity: Final = client.poll_daily_activity_for_key( + token, start=started - timedelta(days=1), end=end, min_requests=1 + ) + assert activity is not None, f"/user/daily/activity never listed key {token} under api_keys" + + exported: Final = client.proxy.transport.probe( + EXPORT_ROUTE, + params=UsageExportParams( + start_date=(started - timedelta(days=1)).strftime("%Y-%m-%d"), + end_date=end.strftime("%Y-%m-%d"), + api_key=token, + ), + ) + assert exported.status_code == 200, f"{EXPORT_ROUTE} answered {exported.status_code}: {exported.body}" + rows: Final = _EXPORT_ROWS.validate_json(exported.body) + key_rows: Final = tuple(row for row in rows if row.api_key == token) + assert key_rows, f"{EXPORT_ROUTE} returned no daily_with_keys row for {token}: {exported.body}" + for row in key_rows: + assert row.user_id == user_id, f"export row for {token} is owned by {row.user_id}, not {user_id}" + assert row.key_metadata.tag == f"cost-center-{marker}", f"key metadata not exported: {row.key_metadata}" + assert row.user_metadata.department == f"research-{marker}", ( + f"user metadata not exported: {row.user_metadata}" + ) diff --git a/tests/integration/spend/_daily_activity_fixtures.py b/tests/integration/spend/_daily_activity_fixtures.py index 8b4f35365ef..15574d88e82 100644 --- a/tests/integration/spend/_daily_activity_fixtures.py +++ b/tests/integration/spend/_daily_activity_fixtures.py @@ -198,8 +198,10 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, ), ) cursor.execute( - sql.SQL("INSERT INTO {} (user_id, user_email, models) VALUES (%s, %s, %s)").format(user_table), - ("user-1", "user@example.com", []), + sql.SQL("INSERT INTO {} (user_id, user_email, models, metadata) VALUES (%s, %s, %s, %s)").format( + user_table + ), + ("user-1", "user@example.com", [], '{"department": "research"}'), ) cursor.execute( sql.SQL("INSERT INTO {} (team_id, team_alias, admins, members, models) VALUES (%s, %s, %s, %s, %s)").format( diff --git a/tests/integration/spend/test_daily_activity_repository.py b/tests/integration/spend/test_daily_activity_repository.py index 1ed85d4a3bc..4c16c5c951b 100644 --- a/tests/integration/spend/test_daily_activity_repository.py +++ b/tests/integration/spend/test_daily_activity_repository.py @@ -236,6 +236,11 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: "user-1", "user@example.com", ) + assert deleted_key_export.key_metadata == {"tags": ["archived"]} + assert deleted_key_export.user_metadata == {"department": "research"} + active_key_export: Final = next(row for row in exports if row.api_key == "key-a") + assert active_key_export.key_metadata == {"tags": ["blue", "gold"]} + assert active_key_export.user_metadata == {"department": "research"} user_exports: Final = tuple( [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY_WITH_USERS)] ) @@ -245,11 +250,14 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: "user@example.com", 273.0, ) + assert user_exports[0].user_metadata == {"department": "research"} + assert user_exports[0].key_metadata == {} daily_export: Final = tuple( [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY)] ) assert len(daily_export) == 1 assert daily_export[0].spend == 1273.0 + assert (daily_export[0].key_metadata, daily_export[0].user_metadata) == ({}, {}) assert daily_export[0].flat_cost == 42.0 metadata: Final = await repository.key_metadata(frozenset(("key-a", "key-target")), None) @@ -684,9 +692,7 @@ async def test_user_page_ranks_users_and_folds_null_ids() -> None: repository: Final = _repository(database) full_page: Final = await repository.user_page(scope, offset=0, limit=50) empty_page: Final = await repository.user_page(scope, offset=full_page.total_users + 1, limit=50) - scoped_page: Final = await repository.user_page( - replace(scope, entity_ids=("user-alpha",)), offset=0, limit=50 - ) + scoped_page: Final = await repository.user_page(replace(scope, entity_ids=("user-alpha",)), offset=0, limit=50) assert tuple((row.user_id, row.spend) for row in full_page.rows) == ( (None, 45.0), diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index e487c00f25d..f39dedb3dcb 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -1,5 +1,6 @@ import csv import io +import json from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from dataclasses import dataclass, fields from itertools import chain @@ -390,6 +391,8 @@ class _FakeRepository: failed_requests=row.failed_requests, cache_read_input_tokens=row.cache_read_input_tokens, cache_creation_input_tokens=row.cache_creation_input_tokens, + key_metadata={"cost_center": "=finance", "tier": 2}, + user_metadata={"department": "research"}, ) @@ -1160,6 +1163,9 @@ def test_export_routes_stream_csv_and_preserve_row_counts( assert len(records) == 7 assert records[1][2] == "'=entity" assert records[1][4] == "'+key" + columns: Final = dict(zip(records[0], records[1], strict=True)) + assert json.loads(columns["key_metadata"]) == {"cost_center": "=finance", "tier": 2} + assert json.loads(columns["user_metadata"]) == {"department": "research"} @pytest.mark.parametrize(("prefix", "query_name", "entity_id"), _ENTITY_CASES) @@ -1180,6 +1186,8 @@ def test_export_routes_stream_json_arrays( records: Final = response.json() assert isinstance(records, list) and len(records) == 6, response.text assert records[0]["entity_alias"] == "=entity" + assert records[0]["key_metadata"] == {"cost_center": "=finance", "tier": 2} + assert records[0]["user_metadata"] == {"department": "research"} def test_export_first_row_error_returns_json_error_before_streaming( diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index f8375b93dad..aed120c85c5 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -451,6 +451,34 @@ def test_export_uses_latest_deleted_key_metadata(export_type: ExportType) -> Non assert "COALESCE(vt.user_id, dvt.user_id)" in query.sql +@pytest.mark.parametrize( + ("export_type", "key_metadata", "user_metadata"), + [ + ( + ExportType.DAILY_WITH_KEYS, + "THEN COALESCE(vt.metadata, dvt.metadata)::text END), '{}')::jsonb AS key_metadata", + "THEN u.metadata::text END), '{}')::jsonb AS user_metadata", + ), + ( + ExportType.DAILY_WITH_USERS, + "'{}'::jsonb AS key_metadata", + "THEN u.metadata::text END), '{}')::jsonb AS user_metadata", + ), + (ExportType.DAILY, "'{}'::jsonb AS key_metadata", "'{}'::jsonb AS user_metadata"), + (ExportType.DAILY_WITH_MODELS, "'{}'::jsonb AS key_metadata", "'{}'::jsonb AS user_metadata"), + ], +) +def test_export_selects_stored_key_and_user_metadata_objects( + export_type: ExportType, key_metadata: str, user_metadata: str +) -> None: + query = build_export_sql(_scope(entity_ids=None), export_type=export_type, after=None, batch_size=2) + + assert key_metadata in query.sql + assert user_metadata in query.sql + if export_type is ExportType.DAILY_WITH_KEYS: + assert "SELECT key_alias, user_id, metadata" in query.sql + + @pytest.mark.parametrize("export_type", tuple(ExportType)) def test_export_without_cursor_omits_cursor_predicate_and_parameters(export_type: ExportType) -> None: query = build_export_sql( From e8e2fad99dbe440ed0d1c42f07800bf652fa1022 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 3 Nov 2026 00:06:07 +0000 Subject: [PATCH 02/11] fix(usage-export): only populate key and user metadata for admin view and strip credential and SCIM slots --- litellm/constants.py | 3 ++ litellm/proxy/common_utils/callback_utils.py | 3 +- .../daily_activity_routes.py | 8 +++- .../internal_user_endpoints.py | 8 +--- .../repositories/daily_activity_repository.py | 14 +++++- litellm/repositories/daily_activity_sql.py | 46 ++++++++++++------- .../proxy/management_endpoints/scim_v2.py | 3 ++ .../coverage_registry/quota_management.yaml | 2 +- .../test_usage_export_metadata_e2e.py | 22 ++++++++- .../spend/_daily_activity_fixtures.py | 19 +++++++- .../spend/test_daily_activity_repository.py | 26 +++++++++-- .../test_daily_activity_routes.py | 43 +++++++++++++++-- .../repositories/test_daily_activity_sql.py | 46 +++++++++++++------ 13 files changed, 190 insertions(+), 53 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index cd7c0176428..34e07d83b35 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2312,6 +2312,9 @@ USAGE_MODEL_TOP_KEYS_MAX: Final[int] = 100 USAGE_CACHE_LEAKAGE_KEYS_DEFAULT: Final[int] = 20 USAGE_CACHE_LEAKAGE_KEYS_MAX: Final[int] = 100 USAGE_EXPORT_BATCH_SIZE: Final[int] = 1000 +CALLBACK_CONFIG_METADATA_SLOTS: Final[frozenset[str]] = frozenset( + {"logging", "callback_settings", "secret_manager_settings"} +) # Furthest back the catch-up pass looks for unpriced PTU days when a deployment # declares no ptu_effective_from, bounding the scan for an open-ended window. PTU_ROLLUP_MAX_BACKFILL_DAYS: Final[int] = 90 diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index d1787f3ff8b..29037859ea1 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -14,6 +14,7 @@ from litellm import get_secret from litellm._logging import verbose_proxy_logger from litellm.constants import ( BAGGAGE_MAPPED_TAGS_METADATA_KEY, + CALLBACK_CONFIG_METADATA_SLOTS, CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH, @@ -52,7 +53,7 @@ _CALLBACK_VAR_ENCRYPTED_PREFIX: Final = CALLBACK_VAR_ENCRYPTED_PREFIX # Metadata slots that hold operator-configured callback and secret-manager setup # (and therefore integration credentials). Resolved from UserAPIKeyAuth during # pre-call setup, never read back off the copies stamped into request metadata. -_CALLBACK_CONFIG_SLOTS: Final = frozenset({"logging", "callback_settings", "secret_manager_settings"}) +_CALLBACK_CONFIG_SLOTS: Final = CALLBACK_CONFIG_METADATA_SLOTS blue_color_code: Final = "\033[94m" reset_color_code: Final = "\033[0m" diff --git a/litellm/proxy/management_endpoints/daily_activity_routes.py b/litellm/proxy/management_endpoints/daily_activity_routes.py index 1abda1aae33..7a1cb7db5f0 100644 --- a/litellm/proxy/management_endpoints/daily_activity_routes.py +++ b/litellm/proxy/management_endpoints/daily_activity_routes.py @@ -12,7 +12,7 @@ from fastapi.responses import StreamingResponse from litellm import constants from litellm._logging import verbose_proxy_logger -from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth, user_api_key_has_admin_view from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.common_daily_activity import ( InvalidDateRange, @@ -497,7 +497,11 @@ def _register_export_route(router: APIRouter, resolver: EntityScopeResolver, pre prisma_client, user_aggregated=False, ) - rows: Final = repository.export_rows(resolved.scope, export_type=export_type) + rows: Final = repository.export_rows( + resolved.scope, + export_type=export_type, + include_metadata=user_api_key_has_admin_view(user_api_key_dict), + ) first_row: Final = await anext(rows, None) return StreamingResponse( _stream_export_rows(first_row, rows, file_format), diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 54a2d4062d2..405794eecb4 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -106,9 +106,7 @@ from litellm.types.proxy.management_endpoints.internal_user_endpoints import ( UserUpdateResult, ) from litellm.types.proxy.management_endpoints.scim_v2 import ( - SCIM_ENTERPRISE_METADATA_KEY, - SCIM_ENTITLEMENTS_METADATA_KEY, - SCIM_ROLES_METADATA_KEY, + SCIM_DIRECTORY_METADATA_KEYS, ) from litellm.types.utils import BudgetConfig @@ -882,9 +880,7 @@ async def _get_user_info_teams( return team_list, teams_1 -_SCIM_DIRECTORY_METADATA_KEYS: Final = frozenset( - {SCIM_ENTERPRISE_METADATA_KEY, SCIM_ENTITLEMENTS_METADATA_KEY, SCIM_ROLES_METADATA_KEY} -) +_SCIM_DIRECTORY_METADATA_KEYS: Final = SCIM_DIRECTORY_METADATA_KEYS def _redact_scim_enterprise_metadata( diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 321468c4976..e60e3cf2a8e 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -258,12 +258,22 @@ class DailyActivityRepository: query: Final = build_cache_leakage_keys_sql(scope, limit=limit) return _KEY_SPEND_ADAPTER.validate_python(await self._query(query)) - async def export_rows(self, scope: DailyActivityScope, *, export_type: ExportType) -> AsyncIterator[ExportRow]: + async def export_rows( + self, scope: DailyActivityScope, *, export_type: ExportType, include_metadata: bool = False + ) -> AsyncIterator[ExportRow]: batch_size: Final = constants.USAGE_EXPORT_BATCH_SIZE cursor: ExportCursor | None = None # rebind-ok: each page advances the export keyset cursor while True: batch: tuple[ExportRow, ...] = _EXPORT_ADAPTER.validate_python( - await self._query(build_export_sql(scope, export_type=export_type, after=cursor, batch_size=batch_size)) + await self._query( + build_export_sql( + scope, + export_type=export_type, + after=cursor, + batch_size=batch_size, + include_metadata=include_metadata, + ) + ) ) for row in batch: yield row diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 5393fc42100..756bba85278 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -8,8 +8,9 @@ from typing import Final from typing_extensions import assert_never from litellm import constants -from litellm.constants import PTU_SENTINEL_API_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL +from litellm.constants import CALLBACK_CONFIG_METADATA_SLOTS, PTU_SENTINEL_API_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.repositories.daily_activity_buckets import UNATTRIBUTED_API_KEY, UNKNOWN_MODEL_GROUP +from litellm.types.proxy.management_endpoints.scim_v2 import SCIM_DIRECTORY_METADATA_KEYS from litellm.types.repositories.daily_activity import DailyActivityScope, DailyActivityTable, ExportType _API_KEY_ROLLED_UP_BIT: Final = 32 @@ -447,11 +448,16 @@ def build_cache_leakage_keys_sql(scope: DailyActivityScope, *, limit: int) -> Sq def build_export_sql( - scope: DailyActivityScope, *, export_type: ExportType, after: ExportCursor | None, batch_size: int + scope: DailyActivityScope, + *, + export_type: ExportType, + after: ExportCursor | None, + batch_size: int, + include_metadata: bool = False, ) -> SqlQuery: _bounded_limit(batch_size) where_clause, where_params = build_where_clause(scope) - group_key, output_key, user_fields, type_joins = _export_grouping(export_type) + group_key, output_key, user_fields, type_joins = _export_grouping(export_type, include_metadata=include_metadata) grouping_keys: Final = ( f"scoped.date, COALESCE(scoped.\"{scope.entity_id_field}\", '')", *((group_key,) if export_type is not ExportType.DAILY else ()), @@ -520,18 +526,26 @@ def build_export_sql( ) -_EMPTY_METADATA_FIELDS: Final = ", '{}'::jsonb AS key_metadata, '{}'::jsonb AS user_metadata" -_USER_METADATA_FIELD: Final = ( - "COALESCE(MIN(CASE WHEN jsonb_typeof(u.metadata) = 'object' THEN u.metadata::text END), '{}')::jsonb " - "AS user_metadata" -) -_KEY_METADATA_FIELD: Final = ( - "COALESCE(MIN(CASE WHEN jsonb_typeof(COALESCE(vt.metadata, dvt.metadata)) = 'object' " - "THEN COALESCE(vt.metadata, dvt.metadata)::text END), '{}')::jsonb AS key_metadata" +def _metadata_object_field(expression: str, redacted_keys: frozenset[str], alias: str) -> str: + redacted: Final = " - ".join((expression, *(f"'{key}'" for key in sorted(redacted_keys)))) + return ( + f"COALESCE(MIN(CASE WHEN jsonb_typeof({expression}) = 'object' THEN ({redacted})::text END), '{{}}')::jsonb " + f"AS {alias}" + ) + + +_EMPTY_KEY_METADATA_FIELD: Final = "'{}'::jsonb AS key_metadata" +_EMPTY_USER_METADATA_FIELD: Final = "'{}'::jsonb AS user_metadata" +_EMPTY_METADATA_FIELDS: Final = f", {_EMPTY_KEY_METADATA_FIELD}, {_EMPTY_USER_METADATA_FIELD}" +_USER_METADATA_FIELD: Final = _metadata_object_field("u.metadata", SCIM_DIRECTORY_METADATA_KEYS, "user_metadata") +_KEY_METADATA_FIELD: Final = _metadata_object_field( + "COALESCE(vt.metadata, dvt.metadata)", CALLBACK_CONFIG_METADATA_SLOTS, "key_metadata" ) -def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str, ...]]: +def _export_grouping(export_type: ExportType, *, include_metadata: bool) -> tuple[str, str, str, tuple[str, ...]]: + key_metadata: Final = _KEY_METADATA_FIELD if include_metadata else _EMPTY_KEY_METADATA_FIELD + user_metadata: Final = _USER_METADATA_FIELD if include_metadata else _EMPTY_USER_METADATA_FIELD if export_type is ExportType.DAILY: return ( "''", @@ -545,10 +559,8 @@ def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str, _API_KEY_EXPR, "MAX(CASE WHEN scoped.api_key = '' THEN 'Unattributed' " "ELSE COALESCE(vt.key_alias, dvt.key_alias) END) AS key_alias, " - "MAX(COALESCE(vt.user_id, dvt.user_id)) AS user_id, MAX(u.user_email) AS user_email, " - + _KEY_METADATA_FIELD - + ", " - + _USER_METADATA_FIELD, + f"MAX(COALESCE(vt.user_id, dvt.user_id)) AS user_id, MAX(u.user_email) AS user_email, " + f"{key_metadata}, {user_metadata}", ( "LEFT JOIN \"LiteLLM_VerificationToken\" vt ON vt.token = scoped.api_key AND scoped.api_key <> ''", """LEFT JOIN LATERAL ( @@ -573,7 +585,7 @@ def _export_grouping(export_type: ExportType) -> tuple[str, str, str, tuple[str, "COALESCE(vt.user_id, dvt.user_id, '')", "NULL::text", "NULL::text AS key_alias, MAX(COALESCE(vt.user_id, dvt.user_id)) AS user_id, " - "MAX(u.user_email) AS user_email, '{}'::jsonb AS key_metadata, " + _USER_METADATA_FIELD, + f"MAX(u.user_email) AS user_email, {_EMPTY_KEY_METADATA_FIELD}, {user_metadata}", ( 'LEFT JOIN "LiteLLM_VerificationToken" vt ON vt.token = scoped.api_key', """LEFT JOIN LATERAL ( diff --git a/litellm/types/proxy/management_endpoints/scim_v2.py b/litellm/types/proxy/management_endpoints/scim_v2.py index fadb309e0cd..dd9df4f9a3e 100644 --- a/litellm/types/proxy/management_endpoints/scim_v2.py +++ b/litellm/types/proxy/management_endpoints/scim_v2.py @@ -18,6 +18,9 @@ SCIM_ENTERPRISE_USER_SCHEMA: Final = "urn:ietf:params:scim:schemas:extension:ent SCIM_ENTERPRISE_METADATA_KEY: Final = "scim_enterprise" SCIM_ENTITLEMENTS_METADATA_KEY: Final = "scim_entitlements" SCIM_ROLES_METADATA_KEY: Final = "scim_roles" +SCIM_DIRECTORY_METADATA_KEYS: Final[frozenset[str]] = frozenset( + {SCIM_ENTERPRISE_METADATA_KEY, SCIM_ENTITLEMENTS_METADATA_KEY, SCIM_ROLES_METADATA_KEY} +) SCIM_MANAGED_TEAM_METADATA_KEY: Final = "scim_managed" SCIM_TEAM_DATA_METADATA_KEY: Final = "scim_data" diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 75dd883742f..56f5e7a66ca 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -40,7 +40,7 @@ - {id: quota_management.budget.fallback.routes_to_fallback, module: quota_management, tier: P1, behavior: budget, variant: fallback, assertions: [routes_to_fallback], exercised_on: [messages], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "budget_fallbacks reroute to the fallback model once the primary's budget is exhausted"} - {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"} - {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"} -- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata], exercised_on: [chat_completions], source: "repositories/daily_activity_sql.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects"} +- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata, empty_metadata_without_admin_view], exercised_on: [chat_completions], source: "repositories/daily_activity_sql.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects for callers with admin view and {} for everyone else"} - {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"} - {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} - {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"} diff --git a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py index e70eba87689..fee0c97d623 100644 --- a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py @@ -4,7 +4,8 @@ A customer charging usage back by cost center keeps that cost center in the virtual key's metadata and the owner's user metadata. Before LIT-8524 the Usage page export only carried the key alias, user id and email, so every exported row needed a /key/info and /user/info round trip to recover the -metadata. The export now joins both objects onto each daily_with_keys row. +metadata. The export now joins both objects onto each daily_with_keys row for +callers with admin view, and returns {} for everyone else. """ from __future__ import annotations @@ -38,6 +39,7 @@ class UsageExportParams(BaseModel): start_date: str end_date: str api_key: str + user_id: str | None = None export_type: str = "daily_with_keys" format: str = "json" @@ -150,3 +152,21 @@ class TestUsageExportMetadata: assert row.user_metadata.department == f"research-{marker}", ( f"user metadata not exported: {row.user_metadata}" ) + + own_view: Final = client.proxy.transport.probe( + EXPORT_ROUTE, + params=UsageExportParams( + start_date=(started - timedelta(days=1)).strftime("%Y-%m-%d"), + end_date=end.strftime("%Y-%m-%d"), + api_key=token, + user_id=user_id, + ), + headers=client.proxy.transport.bearer(record.key), + ) + assert own_view.status_code == 200, f"{EXPORT_ROUTE} answered {own_view.status_code}: {own_view.body}" + own_rows: Final = _EXPORT_ROWS.validate_json(own_view.body) + assert own_rows, f"{EXPORT_ROUTE} returned no rows for the key's own user: {own_view.body}" + for row in own_rows: + assert (row.key_metadata, row.user_metadata) == (UsageExportKeyMetadata(), UsageExportUserMetadata()), ( + f"a caller without admin view got metadata objects: {row}" + ) diff --git a/tests/integration/spend/_daily_activity_fixtures.py b/tests/integration/spend/_daily_activity_fixtures.py index 15574d88e82..a7e4c0df918 100644 --- a/tests/integration/spend/_daily_activity_fixtures.py +++ b/tests/integration/spend/_daily_activity_fixtures.py @@ -171,7 +171,16 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, "INSERT INTO {} (token, key_alias, team_id, user_id, metadata, models) VALUES (%s, %s, %s, %s, %s, %s)" ).format(verification_token_table), ( - ("key-a", "alias-a", "team-1", "user-1", '{"tags": ["blue", "gold"]}', []), + ( + "key-a", + "alias-a", + "team-1", + "user-1", + '{"tags": ["blue", "gold"], "cost_center": "cc-1", ' + '"logging": [{"callback_name": "langfuse", "callback_vars": {"langfuse_secret_key": "sk"}}], ' + '"callback_settings": {"otel": {"headers": "x"}}, "secret_manager_settings": {"vault": "v"}}', + [], + ), ("key-b", "alias-b", "team-1", "user-1", '{"tags": []}', []), ("key-c", "alias-c", "team-1", "user-1", '{"tags": []}', []), ("key-cache", "alias-cache", "team-1", "user-1", '{"tags": []}', []), @@ -201,7 +210,13 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, sql.SQL("INSERT INTO {} (user_id, user_email, models, metadata) VALUES (%s, %s, %s, %s)").format( user_table ), - ("user-1", "user@example.com", [], '{"department": "research"}'), + ( + "user-1", + "user@example.com", + [], + '{"department": "research", "scim_enterprise": {"employeeNumber": "E-1"}, ' + '"scim_entitlements": ["ent"], "scim_roles": ["role"]}', + ), ) cursor.execute( sql.SQL("INSERT INTO {} (team_id, team_alias, admins, members, models) VALUES (%s, %s, %s, %s, %s)").format( diff --git a/tests/integration/spend/test_daily_activity_repository.py b/tests/integration/spend/test_daily_activity_repository.py index 4c16c5c951b..6229bbc13be 100644 --- a/tests/integration/spend/test_daily_activity_repository.py +++ b/tests/integration/spend/test_daily_activity_repository.py @@ -219,7 +219,12 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: assert tuple(row.api_key for row in leakage_keys) == ("key-cache", "key-c") exports: Final = tuple( - [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY_WITH_KEYS)] + [ + row + async for row in repository.export_rows( + team_scope, export_type=ExportType.DAILY_WITH_KEYS, include_metadata=True + ) + ] ) assert tuple(row.api_key for row in exports) == ( "key-a", @@ -239,10 +244,20 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: assert deleted_key_export.key_metadata == {"tags": ["archived"]} assert deleted_key_export.user_metadata == {"department": "research"} active_key_export: Final = next(row for row in exports if row.api_key == "key-a") - assert active_key_export.key_metadata == {"tags": ["blue", "gold"]} + assert active_key_export.key_metadata == {"tags": ["blue", "gold"], "cost_center": "cc-1"} assert active_key_export.user_metadata == {"department": "research"} + unprivileged_exports: Final = tuple( + [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY_WITH_KEYS)] + ) + assert tuple(row.api_key for row in unprivileged_exports) == tuple(row.api_key for row in exports) + assert all((row.key_metadata, row.user_metadata) == ({}, {}) for row in unprivileged_exports) user_exports: Final = tuple( - [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY_WITH_USERS)] + [ + row + async for row in repository.export_rows( + team_scope, export_type=ExportType.DAILY_WITH_USERS, include_metadata=True + ) + ] ) assert len(user_exports) == 1 assert (user_exports[0].user_id, user_exports[0].user_email, user_exports[0].spend) == ( @@ -253,7 +268,10 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: assert user_exports[0].user_metadata == {"department": "research"} assert user_exports[0].key_metadata == {} daily_export: Final = tuple( - [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY)] + [ + row + async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY, include_metadata=True) + ] ) assert len(daily_export) == 1 assert daily_export[0].spend == 1273.0 diff --git a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py index f39dedb3dcb..bb6cf8196c6 100644 --- a/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py +++ b/tests/unit/proxy/management_endpoints/test_daily_activity_routes.py @@ -260,6 +260,7 @@ class _FakeRepository: self.user_page_call: tuple[DailyActivityScope, int, int] | None = None self.user_metadata = AsyncMock(side_effect=self._user_metadata) self.export_rows_error: Exception | None = None + self.export_rows_include_metadata: bool | None = None def _matching_rows(self, scope: DailyActivityScope) -> tuple[_Activity, ...]: return tuple( @@ -369,7 +370,10 @@ class _FakeRepository: for key in api_keys } - async def export_rows(self, scope: DailyActivityScope, *, export_type: ExportType) -> AsyncIterator[ExportRow]: + async def export_rows( + self, scope: DailyActivityScope, *, export_type: ExportType, include_metadata: bool = False + ) -> AsyncIterator[ExportRow]: + self.export_rows_include_metadata = include_metadata if self.export_rows_error is not None: raise self.export_rows_error for row in self._matching_rows(scope): @@ -391,8 +395,8 @@ class _FakeRepository: failed_requests=row.failed_requests, cache_read_input_tokens=row.cache_read_input_tokens, cache_creation_input_tokens=row.cache_creation_input_tokens, - key_metadata={"cost_center": "=finance", "tier": 2}, - user_metadata={"department": "research"}, + key_metadata={"cost_center": "=finance", "tier": 2} if include_metadata else {}, + user_metadata={"department": "research"} if include_metadata else {}, ) @@ -1190,6 +1194,39 @@ def test_export_routes_stream_json_arrays( assert records[0]["user_metadata"] == {"department": "research"} +@pytest.mark.parametrize( + ("role", "expects_metadata"), + [ + (LitellmUserRoles.PROXY_ADMIN, True), + (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, True), + (LitellmUserRoles.INTERNAL_USER, False), + ], +) +def test_export_metadata_objects_only_for_callers_with_admin_view( + daily_activity_client: tuple[TestClient, _FakeRepository], + role: LitellmUserRoles, + expects_metadata: bool, +) -> None: + client, repository = daily_activity_client + response: Final = client.get( + "/user/daily/activity/export", + params={ + **_entity_params("user_id", "user-a"), + "export_type": ExportType.DAILY_WITH_KEYS.value, + "format": "json", + }, + headers={"x-user-role": role.value, "x-user-id": "user-a"}, + ) + assert response.status_code == 200, response.text + assert repository.export_rows_include_metadata is expects_metadata + records: Final = response.json() + assert records, response.text + expected_key: Final = {"cost_center": "=finance", "tier": 2} if expects_metadata else {} + expected_user: Final = {"department": "research"} if expects_metadata else {} + assert all(record["key_metadata"] == expected_key for record in records), response.text + assert all(record["user_metadata"] == expected_user for record in records), response.text + + def test_export_first_row_error_returns_json_error_before_streaming( daily_activity_client: tuple[TestClient, _FakeRepository], ) -> None: diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index aed120c85c5..d107d4e7e09 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -451,27 +451,33 @@ def test_export_uses_latest_deleted_key_metadata(export_type: ExportType) -> Non assert "COALESCE(vt.user_id, dvt.user_id)" in query.sql +_REDACTED_KEY_METADATA: Final = ( + "THEN (COALESCE(vt.metadata, dvt.metadata) - 'callback_settings' - 'logging' - 'secret_manager_settings')" + "::text END), '{}')::jsonb AS key_metadata" +) +_REDACTED_USER_METADATA: Final = ( + "THEN (u.metadata - 'scim_enterprise' - 'scim_entitlements' - 'scim_roles')::text END), '{}')::jsonb " + "AS user_metadata" +) +_EMPTY_KEY_METADATA: Final = "'{}'::jsonb AS key_metadata" +_EMPTY_USER_METADATA: Final = "'{}'::jsonb AS user_metadata" + + @pytest.mark.parametrize( ("export_type", "key_metadata", "user_metadata"), [ - ( - ExportType.DAILY_WITH_KEYS, - "THEN COALESCE(vt.metadata, dvt.metadata)::text END), '{}')::jsonb AS key_metadata", - "THEN u.metadata::text END), '{}')::jsonb AS user_metadata", - ), - ( - ExportType.DAILY_WITH_USERS, - "'{}'::jsonb AS key_metadata", - "THEN u.metadata::text END), '{}')::jsonb AS user_metadata", - ), - (ExportType.DAILY, "'{}'::jsonb AS key_metadata", "'{}'::jsonb AS user_metadata"), - (ExportType.DAILY_WITH_MODELS, "'{}'::jsonb AS key_metadata", "'{}'::jsonb AS user_metadata"), + (ExportType.DAILY_WITH_KEYS, _REDACTED_KEY_METADATA, _REDACTED_USER_METADATA), + (ExportType.DAILY_WITH_USERS, _EMPTY_KEY_METADATA, _REDACTED_USER_METADATA), + (ExportType.DAILY, _EMPTY_KEY_METADATA, _EMPTY_USER_METADATA), + (ExportType.DAILY_WITH_MODELS, _EMPTY_KEY_METADATA, _EMPTY_USER_METADATA), ], ) -def test_export_selects_stored_key_and_user_metadata_objects( +def test_export_selects_redacted_key_and_user_metadata_objects_for_admin_view( export_type: ExportType, key_metadata: str, user_metadata: str ) -> None: - query = build_export_sql(_scope(entity_ids=None), export_type=export_type, after=None, batch_size=2) + query = build_export_sql( + _scope(entity_ids=None), export_type=export_type, after=None, batch_size=2, include_metadata=True + ) assert key_metadata in query.sql assert user_metadata in query.sql @@ -479,6 +485,18 @@ def test_export_selects_stored_key_and_user_metadata_objects( assert "SELECT key_alias, user_id, metadata" in query.sql +@pytest.mark.parametrize("export_type", tuple(ExportType)) +def test_export_without_metadata_selects_empty_objects_and_never_reads_metadata_columns( + export_type: ExportType, +) -> None: + query = build_export_sql(_scope(entity_ids=None), export_type=export_type, after=None, batch_size=2) + + assert _EMPTY_KEY_METADATA in query.sql + assert _EMPTY_USER_METADATA in query.sql + assert "u.metadata" not in query.sql + assert "vt.metadata" not in query.sql + + @pytest.mark.parametrize("export_type", tuple(ExportType)) def test_export_without_cursor_omits_cursor_predicate_and_parameters(export_type: ExportType) -> None: query = build_export_sql( From e964397a2243b80a8ba1654efc02158f82bfd4a4 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 3 Nov 2026 19:00:23 +0000 Subject: [PATCH 03/11] feat(usage-export): redact credential-named values inside exported key and user metadata --- .../repositories/daily_activity_repository.py | 14 +++++++- .../coverage_registry/quota_management.yaml | 2 +- tests/e2e/models.py | 1 + .../test_usage_export_metadata_e2e.py | 9 +++-- .../spend/_daily_activity_fixtures.py | 4 +-- .../spend/test_daily_activity_repository.py | 12 ++++--- .../test_daily_activity_repository.py | 33 +++++++++++++++++++ 7 files changed, 65 insertions(+), 10 deletions(-) diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index e60e3cf2a8e..0527def01a5 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -1,5 +1,6 @@ import asyncio from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import replace from datetime import datetime from itertools import groupby from types import MappingProxyType @@ -11,6 +12,7 @@ from typing_extensions import assert_never from litellm import constants from litellm._logging import verbose_proxy_logger from litellm.constants import UNKNOWN_MODEL_SPEND_LOG_MODEL +from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.repositories.chunked_in import find_many_in from litellm.repositories.daily_activity_sql import ( ExportCursor, @@ -140,6 +142,16 @@ def _daily_rows_table( raise AssertionError("unreachable") +def _with_redacted_metadata(row: ExportRow) -> ExportRow: + if not row.key_metadata and not row.user_metadata: + return row + return replace( + row, + key_metadata=redact_credentials_in_payload(row.key_metadata), + user_metadata=redact_credentials_in_payload(row.user_metadata), + ) + + def _next_export_cursor(batch: tuple[ExportRow, ...], export_type: ExportType) -> ExportCursor: last: Final = batch[-1] cursor_key: Final = ( @@ -276,7 +288,7 @@ class DailyActivityRepository: ) ) for row in batch: - yield row + yield _with_redacted_metadata(row) if len(batch) < batch_size: return cursor = _next_export_cursor(batch, export_type) diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 56f5e7a66ca..0659de6febe 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -40,7 +40,7 @@ - {id: quota_management.budget.fallback.routes_to_fallback, module: quota_management, tier: P1, behavior: budget, variant: fallback, assertions: [routes_to_fallback], exercised_on: [messages], source: "proxy/hooks/model_max_budget_limiter.py", rationale: "budget_fallbacks reroute to the fallback model once the primary's budget is exhausted"} - {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"} - {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"} -- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata, empty_metadata_without_admin_view], exercised_on: [chat_completions], source: "repositories/daily_activity_sql.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects for callers with admin view and {} for everyone else"} +- {id: quota_management.spend_tracking.usage_export.carries_key_and_user_metadata, module: quota_management, tier: P1, behavior: spend_tracking, variant: usage_export, assertions: [carries_key_and_user_metadata, empty_metadata_without_admin_view, redacts_credential_named_values], exercised_on: [chat_completions], source: "repositories/daily_activity_repository.py / LIT-8524", rationale: "The daily_with_keys usage export row carries the key's and the owning user's stored metadata objects for callers with admin view with credential-named values replaced by REDACTED, and {} for everyone else"} - {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"} - {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"} - {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 679195276fc..7467d2fd085 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -59,6 +59,7 @@ class KeyMetadata(BaseModel): batch_enqueued_token_limit: int | None = None tag: str | None = None guardrails: list[str] | None = None + webhook_token: str | None = None class ObjectPermission(BaseModel): diff --git a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py index fee0c97d623..a1f5633f17e 100644 --- a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py @@ -46,10 +46,12 @@ class UsageExportParams(BaseModel): class UsageExportKeyMetadata(BaseModel): tag: str | None = None + webhook_token: str | None = None class UsageExportUserMetadata(BaseModel): department: str | None = None + ldap_password: str | None = None class UsageExportRow(BaseModel): @@ -99,7 +101,7 @@ class TestUsageExportMetadata: user_email=f"e2e-usage-export-{marker}@example.com", user_role="internal_user", user_id=f"e2e-usage-export-{marker}", - metadata={"department": f"research-{marker}"}, + metadata={"department": f"research-{marker}", "ldap_password": f"pw-{marker}"}, ), response_type=UserNewResponse, ) @@ -110,7 +112,7 @@ class TestUsageExportMetadata: user_id=user_id, models=[model], key_alias=f"e2e-usage-export-{marker}", - metadata=KeyMetadata(tag=f"cost-center-{marker}"), + metadata=KeyMetadata(tag=f"cost-center-{marker}", webhook_token=f"whsec-{marker}"), ) ) resources.defer(lambda: client.proxy.delete_key(record.key)) @@ -152,6 +154,9 @@ class TestUsageExportMetadata: assert row.user_metadata.department == f"research-{marker}", ( f"user metadata not exported: {row.user_metadata}" ) + assert (row.key_metadata.webhook_token, row.user_metadata.ldap_password) == ("REDACTED", "REDACTED"), ( + f"credential-named metadata values reached the export unmasked: {row}" + ) own_view: Final = client.proxy.transport.probe( EXPORT_ROUTE, diff --git a/tests/integration/spend/_daily_activity_fixtures.py b/tests/integration/spend/_daily_activity_fixtures.py index a7e4c0df918..85c434c6877 100644 --- a/tests/integration/spend/_daily_activity_fixtures.py +++ b/tests/integration/spend/_daily_activity_fixtures.py @@ -176,7 +176,7 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, "alias-a", "team-1", "user-1", - '{"tags": ["blue", "gold"], "cost_center": "cc-1", ' + '{"tags": ["blue", "gold"], "cost_center": "cc-1", "webhook_token": "whsec_1", ' '"logging": [{"callback_name": "langfuse", "callback_vars": {"langfuse_secret_key": "sk"}}], ' '"callback_settings": {"otel": {"headers": "x"}}, "secret_manager_settings": {"vault": "v"}}', [], @@ -214,7 +214,7 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, "user-1", "user@example.com", [], - '{"department": "research", "scim_enterprise": {"employeeNumber": "E-1"}, ' + '{"department": "research", "ldap_password": "pw", "scim_enterprise": {"employeeNumber": "E-1"}, ' '"scim_entitlements": ["ent"], "scim_roles": ["role"]}', ), ) diff --git a/tests/integration/spend/test_daily_activity_repository.py b/tests/integration/spend/test_daily_activity_repository.py index 6229bbc13be..3df2b934112 100644 --- a/tests/integration/spend/test_daily_activity_repository.py +++ b/tests/integration/spend/test_daily_activity_repository.py @@ -242,10 +242,14 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: "user@example.com", ) assert deleted_key_export.key_metadata == {"tags": ["archived"]} - assert deleted_key_export.user_metadata == {"department": "research"} + assert deleted_key_export.user_metadata == {"department": "research", "ldap_password": "REDACTED"} active_key_export: Final = next(row for row in exports if row.api_key == "key-a") - assert active_key_export.key_metadata == {"tags": ["blue", "gold"], "cost_center": "cc-1"} - assert active_key_export.user_metadata == {"department": "research"} + assert active_key_export.key_metadata == { + "tags": ["blue", "gold"], + "cost_center": "cc-1", + "webhook_token": "REDACTED", + } + assert active_key_export.user_metadata == {"department": "research", "ldap_password": "REDACTED"} unprivileged_exports: Final = tuple( [row async for row in repository.export_rows(team_scope, export_type=ExportType.DAILY_WITH_KEYS)] ) @@ -265,7 +269,7 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: "user@example.com", 273.0, ) - assert user_exports[0].user_metadata == {"department": "research"} + assert user_exports[0].user_metadata == {"department": "research", "ldap_password": "REDACTED"} assert user_exports[0].key_metadata == {} daily_export: Final = tuple( [ diff --git a/tests/unit/repositories/test_daily_activity_repository.py b/tests/unit/repositories/test_daily_activity_repository.py index d93a75476a2..ddcc0f0ff11 100644 --- a/tests/unit/repositories/test_daily_activity_repository.py +++ b/tests/unit/repositories/test_daily_activity_repository.py @@ -678,6 +678,39 @@ async def test_daily_rows_exclusion_without_entity_filter_keeps_null_entity_rows assert database.litellm_dailyteamspend.find_many_calls == [expected_where] +@pytest.mark.asyncio +async def test_export_redacts_credential_named_values_inside_metadata_objects() -> None: + database = _FakeDatabase( + ( + ( + { + **_export_row("key-1"), + "key_metadata": { + "cost_center": "cc-1", + "api_key": "sk-live-1234567890", + "webhook": {"url": "https://hooks.example", "auth_token": "t0k3n"}, + "owners": [{"name": "ops", "password": "pw"}], + }, + "user_metadata": {"department": "research", "github_token": "ghp_secret"}, + }, + {**_export_row("key-2"), "key_metadata": {}, "user_metadata": {}}, + ), + ) + ) + repository, _ = _repository(database) + + rows = [row async for row in repository.export_rows(_scope(), export_type=ExportType.DAILY_WITH_KEYS)] + + assert rows[0].key_metadata == { + "cost_center": "cc-1", + "api_key": "REDACTED", + "webhook": {"url": "https://hooks.example", "auth_token": "REDACTED"}, + "owners": [{"name": "ops", "password": "REDACTED"}], + } + assert rows[0].user_metadata == {"department": "research", "github_token": "REDACTED"} + assert (rows[1].key_metadata, rows[1].user_metadata) == ({}, {}) + + @pytest.mark.asyncio async def test_export_is_lazy_and_uses_the_last_row_as_the_next_cursor(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(constants, "USAGE_EXPORT_BATCH_SIZE", 2) From e374b32825fa554f2267c98e9040da57c7728a4d Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 3 Nov 2026 19:04:45 +0000 Subject: [PATCH 04/11] test(usage-export): freeze e2e models, compare raw non-admin metadata, mark query locals Final --- .../test_usage_export_metadata_e2e.py | 18 ++++++++++++------ .../repositories/test_daily_activity_sql.py | 10 +++++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py index a1f5633f17e..2e27dbc9a0d 100644 --- a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py @@ -35,7 +35,7 @@ pytestmark = pytest.mark.e2e EXPORT_ROUTE: Final = "/user/daily/activity/export" -class UsageExportParams(BaseModel): +class UsageExportParams(BaseModel, frozen=True): start_date: str end_date: str api_key: str @@ -44,24 +44,30 @@ class UsageExportParams(BaseModel): format: str = "json" -class UsageExportKeyMetadata(BaseModel): +class UsageExportKeyMetadata(BaseModel, frozen=True): tag: str | None = None webhook_token: str | None = None -class UsageExportUserMetadata(BaseModel): +class UsageExportUserMetadata(BaseModel, frozen=True): department: str | None = None ldap_password: str | None = None -class UsageExportRow(BaseModel): +class UsageExportRow(BaseModel, frozen=True): api_key: str | None = None user_id: str | None = None key_metadata: UsageExportKeyMetadata user_metadata: UsageExportUserMetadata +class UsageExportRawMetadataRow(BaseModel, frozen=True): + key_metadata: dict[str, object] + user_metadata: dict[str, object] + + _EXPORT_ROWS: Final = TypeAdapter(list[UsageExportRow]) +_RAW_METADATA_ROWS: Final = TypeAdapter(list[UsageExportRawMetadataRow]) class TestUsageExportMetadata: @@ -169,9 +175,9 @@ class TestUsageExportMetadata: headers=client.proxy.transport.bearer(record.key), ) assert own_view.status_code == 200, f"{EXPORT_ROUTE} answered {own_view.status_code}: {own_view.body}" - own_rows: Final = _EXPORT_ROWS.validate_json(own_view.body) + own_rows: Final = _RAW_METADATA_ROWS.validate_json(own_view.body) assert own_rows, f"{EXPORT_ROUTE} returned no rows for the key's own user: {own_view.body}" for row in own_rows: - assert (row.key_metadata, row.user_metadata) == (UsageExportKeyMetadata(), UsageExportUserMetadata()), ( + assert (row.key_metadata, row.user_metadata) == ({}, {}), ( f"a caller without admin view got metadata objects: {row}" ) diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index d107d4e7e09..7257fd207a3 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -413,7 +413,7 @@ def test_query_builders_reject_nonpositive_limits(builder) -> None: def test_export_groups_by_requested_key_and_binds_cursor_after_scope( export_type: ExportType, group_key: str, key_filter: str, joins: str ) -> None: - query = build_export_sql( + query: Final = build_export_sql( _scope(entity_ids=None), export_type=export_type, after=ExportCursor(date="2026-01-12", entity_id="user-2", group_key="group-3"), @@ -444,7 +444,7 @@ def test_export_groups_by_requested_key_and_binds_cursor_after_scope( @pytest.mark.parametrize("export_type", [ExportType.DAILY_WITH_KEYS, ExportType.DAILY_WITH_USERS]) def test_export_uses_latest_deleted_key_metadata(export_type: ExportType) -> None: - query = build_export_sql(_scope(entity_ids=None), export_type=export_type, after=None, batch_size=2) + query: Final = build_export_sql(_scope(entity_ids=None), export_type=export_type, after=None, batch_size=2) assert 'FROM "LiteLLM_DeletedVerificationToken"' in query.sql assert "ORDER BY deleted_at DESC" in query.sql @@ -475,7 +475,7 @@ _EMPTY_USER_METADATA: Final = "'{}'::jsonb AS user_metadata" def test_export_selects_redacted_key_and_user_metadata_objects_for_admin_view( export_type: ExportType, key_metadata: str, user_metadata: str ) -> None: - query = build_export_sql( + query: Final = build_export_sql( _scope(entity_ids=None), export_type=export_type, after=None, batch_size=2, include_metadata=True ) @@ -489,7 +489,7 @@ def test_export_selects_redacted_key_and_user_metadata_objects_for_admin_view( def test_export_without_metadata_selects_empty_objects_and_never_reads_metadata_columns( export_type: ExportType, ) -> None: - query = build_export_sql(_scope(entity_ids=None), export_type=export_type, after=None, batch_size=2) + query: Final = build_export_sql(_scope(entity_ids=None), export_type=export_type, after=None, batch_size=2) assert _EMPTY_KEY_METADATA in query.sql assert _EMPTY_USER_METADATA in query.sql @@ -499,7 +499,7 @@ def test_export_without_metadata_selects_empty_objects_and_never_reads_metadata_ @pytest.mark.parametrize("export_type", tuple(ExportType)) def test_export_without_cursor_omits_cursor_predicate_and_parameters(export_type: ExportType) -> None: - query = build_export_sql( + query: Final = build_export_sql( _scope(entity_ids=None), export_type=export_type, after=None, From 901fb3087113c89636f95384a9439fcba4525da9 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 3 Nov 2026 19:31:23 +0000 Subject: [PATCH 05/11] test(usage-export): mark redaction test locals Final --- tests/unit/repositories/test_daily_activity_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/repositories/test_daily_activity_repository.py b/tests/unit/repositories/test_daily_activity_repository.py index ddcc0f0ff11..c4021725cd7 100644 --- a/tests/unit/repositories/test_daily_activity_repository.py +++ b/tests/unit/repositories/test_daily_activity_repository.py @@ -680,7 +680,7 @@ async def test_daily_rows_exclusion_without_entity_filter_keeps_null_entity_rows @pytest.mark.asyncio async def test_export_redacts_credential_named_values_inside_metadata_objects() -> None: - database = _FakeDatabase( + database: Final = _FakeDatabase( ( ( { @@ -699,7 +699,7 @@ async def test_export_redacts_credential_named_values_inside_metadata_objects() ) repository, _ = _repository(database) - rows = [row async for row in repository.export_rows(_scope(), export_type=ExportType.DAILY_WITH_KEYS)] + rows: Final = [row async for row in repository.export_rows(_scope(), export_type=ExportType.DAILY_WITH_KEYS)] assert rows[0].key_metadata == { "cost_center": "cc-1", From 15aad0c17d57740e0f442128c8b0fd69592f4c97 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 3 Nov 2026 22:54:57 +0000 Subject: [PATCH 06/11] fix(usage-export): drop callback and SCIM slots from both metadata objects at any depth and mask camelCase credential names --- .../sensitive_data_masker.py | 63 ++++++++++++------- .../repositories/daily_activity_repository.py | 14 +++-- litellm/repositories/daily_activity_sql.py | 5 +- tests/e2e/models.py | 1 + .../test_usage_export_metadata_e2e.py | 20 ++++-- .../spend/_daily_activity_fixtures.py | 4 +- .../spend/test_daily_activity_repository.py | 1 + .../test_sensitive_data_masker.py | 42 +++++++++++++ .../test_daily_activity_repository.py | 22 +++++-- .../repositories/test_daily_activity_sql.py | 12 ++-- 10 files changed, 139 insertions(+), 45 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 8c402fe9b44..2cf8899cf3a 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -1,3 +1,4 @@ +import re from collections.abc import ItemsView, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass, field @@ -30,6 +31,9 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( ) +_CAMEL_BOUNDARY: Final = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") + + def _object_items(data: Mapping[str, object]) -> ItemsView[str, object]: return data.items() @@ -51,7 +55,9 @@ class SensitiveDataMasker: # If any key segment matches one of these, the key is not considered sensitive # even if it also matches a sensitive pattern. For example, "input_cost_per_token" # contains "token" but "cost" overrides that — it's a pricing field, not a secret. - self.non_sensitive_overrides = non_sensitive_overrides or frozenset(("cost",)) + self.non_sensitive_overrides = ( + non_sensitive_overrides if non_sensitive_overrides is not None else frozenset(("cost",)) + ) self.visible_prefix = visible_prefix self.visible_suffix = visible_suffix @@ -82,10 +88,10 @@ class SensitiveDataMasker: if excluded_keys and key in excluded_keys: return False - key_lower: Final = str(key).lower() - # Split on underscores/hyphens and check if any segment matches the pattern + # Split on underscores, hyphens and camelCase boundaries and check if any segment matches the pattern # This avoids false positives like "max_tokens" matching "token" - # but still catches "api_key", "access_token", etc. + # but still catches "api_key", "access_token", "apiKey", "clientSecret", etc. + key_lower: Final = _CAMEL_BOUNDARY.sub("_", str(key)).lower() key_segments: Final = key_lower.replace("-", "_").split("_") # If any segment matches a non-sensitive override, the key is not sensitive. @@ -255,10 +261,16 @@ def mask_sensitive_keys(data: Mapping[str, object], sensitive_fields: set[str]) return masked -def redact_credentials_in_payload(data: Mapping[str, object]) -> Mapping[str, object]: +def redact_credentials_in_payload( + data: Mapping[str, object], + masker: SensitiveDataMasker = _default_masker, + *, + dropped_keys: AbstractSet[str] = frozenset(), +) -> Mapping[str, object]: """Return a copy of ``data`` where every value under a credential-named key is - replaced by the shared ``REDACTED`` marker, nested mappings are recursed into, - and every other value is preserved by identity. + replaced by the shared ``REDACTED`` marker, every entry whose key is in + ``dropped_keys`` is removed outright at any depth, nested mappings are recursed + into, and every other value is preserved by identity. Sensitive-key detection is delegated to the shared :class:`SensitiveDataMasker`, so the credential names stay in one place. Unlike @@ -271,28 +283,31 @@ def redact_credentials_in_payload(data: Mapping[str, object]) -> Mapping[str, ob rather than passed through, so burying a credential deeper than the walk goes hides it instead of exposing it. """ - return _redact_mapping(data, 0) + return _PayloadRedactor(masker, frozenset(dropped_keys)).mapping(data, 0) -def _redact_mapping(data: Mapping[str, object], depth: int) -> Mapping[str, object]: - return {key: _redact_entry(key, value, depth) for key, value in data.items()} +@dataclass(frozen=True, slots=True) +class _PayloadRedactor: + masker: SensitiveDataMasker + dropped_keys: frozenset[str] + def mapping(self, data: Mapping[str, object], depth: int) -> Mapping[str, object]: + return {key: self.entry(key, value, depth) for key, value in data.items() if key not in self.dropped_keys} -def _redact_entry(key: str, value: object, depth: int) -> object: - if value is not None and _default_masker.is_sensitive_key(key): - return REDACTED - if not isinstance(value, (Mapping, list, tuple)): - return value - if depth >= DEFAULT_MAX_RECURSE_DEPTH: - return REDACTED - if isinstance(value, Mapping): - return _redact_mapping(value, depth + 1) - return _redact_sequence(value, depth + 1) + def entry(self, key: str, value: object, depth: int) -> object: + if value is not None and self.masker.is_sensitive_key(key): + return REDACTED + if not isinstance(value, (Mapping, list, tuple)): + return value + if depth >= DEFAULT_MAX_RECURSE_DEPTH: + return REDACTED + if isinstance(value, Mapping): + return self.mapping(value, depth + 1) + return self.sequence(value, depth + 1) - -def _redact_sequence(values: Sequence[object], depth: int) -> Sequence[object]: - redacted: Final = tuple(_redact_entry("", item, depth) for item in values) - return redacted if isinstance(values, tuple) else list(redacted) + def sequence(self, values: Sequence[object], depth: int) -> Sequence[object]: + redacted: Final = tuple(self.entry("", item, depth) for item in values) + return redacted if isinstance(values, tuple) else list(redacted) # Usage example: diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index 0527def01a5..ae4d9992036 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -12,9 +12,10 @@ from typing_extensions import assert_never from litellm import constants from litellm._logging import verbose_proxy_logger from litellm.constants import UNKNOWN_MODEL_SPEND_LOG_MODEL -from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker, redact_credentials_in_payload from litellm.repositories.chunked_in import find_many_in from litellm.repositories.daily_activity_sql import ( + EXPORT_METADATA_DROPPED_KEYS, ExportCursor, SqlQuery, adjust_dates_for_timezone, @@ -142,13 +143,18 @@ def _daily_rows_table( raise AssertionError("unreachable") +_EXPORT_METADATA_MASKER: Final = SensitiveDataMasker(non_sensitive_overrides=frozenset()) + + +def _redacted_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]: + return redact_credentials_in_payload(metadata, _EXPORT_METADATA_MASKER, dropped_keys=EXPORT_METADATA_DROPPED_KEYS) + + def _with_redacted_metadata(row: ExportRow) -> ExportRow: if not row.key_metadata and not row.user_metadata: return row return replace( - row, - key_metadata=redact_credentials_in_payload(row.key_metadata), - user_metadata=redact_credentials_in_payload(row.user_metadata), + row, key_metadata=_redacted_metadata(row.key_metadata), user_metadata=_redacted_metadata(row.user_metadata) ) diff --git a/litellm/repositories/daily_activity_sql.py b/litellm/repositories/daily_activity_sql.py index 756bba85278..055135b22c0 100644 --- a/litellm/repositories/daily_activity_sql.py +++ b/litellm/repositories/daily_activity_sql.py @@ -537,9 +537,10 @@ def _metadata_object_field(expression: str, redacted_keys: frozenset[str], alias _EMPTY_KEY_METADATA_FIELD: Final = "'{}'::jsonb AS key_metadata" _EMPTY_USER_METADATA_FIELD: Final = "'{}'::jsonb AS user_metadata" _EMPTY_METADATA_FIELDS: Final = f", {_EMPTY_KEY_METADATA_FIELD}, {_EMPTY_USER_METADATA_FIELD}" -_USER_METADATA_FIELD: Final = _metadata_object_field("u.metadata", SCIM_DIRECTORY_METADATA_KEYS, "user_metadata") +EXPORT_METADATA_DROPPED_KEYS: Final[frozenset[str]] = CALLBACK_CONFIG_METADATA_SLOTS | SCIM_DIRECTORY_METADATA_KEYS +_USER_METADATA_FIELD: Final = _metadata_object_field("u.metadata", EXPORT_METADATA_DROPPED_KEYS, "user_metadata") _KEY_METADATA_FIELD: Final = _metadata_object_field( - "COALESCE(vt.metadata, dvt.metadata)", CALLBACK_CONFIG_METADATA_SLOTS, "key_metadata" + "COALESCE(vt.metadata, dvt.metadata)", EXPORT_METADATA_DROPPED_KEYS, "key_metadata" ) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 7467d2fd085..776faf33966 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -60,6 +60,7 @@ class KeyMetadata(BaseModel): tag: str | None = None guardrails: list[str] | None = None webhook_token: str | None = None + apiKey: str | None = None class ObjectPermission(BaseModel): diff --git a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py index 2e27dbc9a0d..6c908915034 100644 --- a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py @@ -47,11 +47,13 @@ class UsageExportParams(BaseModel, frozen=True): class UsageExportKeyMetadata(BaseModel, frozen=True): tag: str | None = None webhook_token: str | None = None + apiKey: str | None = None class UsageExportUserMetadata(BaseModel, frozen=True): department: str | None = None ldap_password: str | None = None + clientSecret: str | None = None class UsageExportRow(BaseModel, frozen=True): @@ -107,7 +109,11 @@ class TestUsageExportMetadata: user_email=f"e2e-usage-export-{marker}@example.com", user_role="internal_user", user_id=f"e2e-usage-export-{marker}", - metadata={"department": f"research-{marker}", "ldap_password": f"pw-{marker}"}, + metadata={ + "department": f"research-{marker}", + "ldap_password": f"pw-{marker}", + "clientSecret": f"cs-{marker}", + }, ), response_type=UserNewResponse, ) @@ -118,7 +124,9 @@ class TestUsageExportMetadata: user_id=user_id, models=[model], key_alias=f"e2e-usage-export-{marker}", - metadata=KeyMetadata(tag=f"cost-center-{marker}", webhook_token=f"whsec-{marker}"), + metadata=KeyMetadata( + tag=f"cost-center-{marker}", webhook_token=f"whsec-{marker}", apiKey=f"ak-{marker}" + ), ) ) resources.defer(lambda: client.proxy.delete_key(record.key)) @@ -160,9 +168,13 @@ class TestUsageExportMetadata: assert row.user_metadata.department == f"research-{marker}", ( f"user metadata not exported: {row.user_metadata}" ) - assert (row.key_metadata.webhook_token, row.user_metadata.ldap_password) == ("REDACTED", "REDACTED"), ( - f"credential-named metadata values reached the export unmasked: {row}" + masked = ( + row.key_metadata.webhook_token, + row.key_metadata.apiKey, + row.user_metadata.ldap_password, + row.user_metadata.clientSecret, ) + assert masked == ("REDACTED",) * 4, f"credential-named metadata values reached the export unmasked: {row}" own_view: Final = client.proxy.transport.probe( EXPORT_ROUTE, diff --git a/tests/integration/spend/_daily_activity_fixtures.py b/tests/integration/spend/_daily_activity_fixtures.py index 85c434c6877..515457126d5 100644 --- a/tests/integration/spend/_daily_activity_fixtures.py +++ b/tests/integration/spend/_daily_activity_fixtures.py @@ -177,6 +177,7 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, "team-1", "user-1", '{"tags": ["blue", "gold"], "cost_center": "cc-1", "webhook_token": "whsec_1", ' + '"scim_roles": ["r"], "langfuse": {"apiKey": "lf", "logging": [{"callback_name": "langfuse"}]}, ' '"logging": [{"callback_name": "langfuse", "callback_vars": {"langfuse_secret_key": "sk"}}], ' '"callback_settings": {"otel": {"headers": "x"}}, "secret_manager_settings": {"vault": "v"}}', [], @@ -214,7 +215,8 @@ def seed_daily_activity_fixture(connection: psycopg.Connection, *, schema: str, "user-1", "user@example.com", [], - '{"department": "research", "ldap_password": "pw", "scim_enterprise": {"employeeNumber": "E-1"}, ' + '{"department": "research", "ldap_password": "pw", "logging": [{"callback_name": "langfuse"}], ' + '"scim_enterprise": {"employeeNumber": "E-1"}, ' '"scim_entitlements": ["ent"], "scim_roles": ["role"]}', ), ) diff --git a/tests/integration/spend/test_daily_activity_repository.py b/tests/integration/spend/test_daily_activity_repository.py index 3df2b934112..d312262a638 100644 --- a/tests/integration/spend/test_daily_activity_repository.py +++ b/tests/integration/spend/test_daily_activity_repository.py @@ -248,6 +248,7 @@ async def test_repository_queries_and_exports_seeded_daily_activity(monkeypatch: "tags": ["blue", "gold"], "cost_center": "cc-1", "webhook_token": "REDACTED", + "langfuse": {"apiKey": "REDACTED"}, } assert active_key_export.user_metadata == {"department": "research", "ldap_password": "REDACTED"} unprivileged_exports: Final = tuple( diff --git a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py index 2182620164c..9039ab99371 100644 --- a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py @@ -612,3 +612,45 @@ def test_redact_credentials_in_payload_leaves_a_realistic_tool_schema_intact(): assert "REDACTED" not in str(result["tools"]) assert result["tools"][0] == tool assert result["api_key"] == "REDACTED" + + +@pytest.mark.parametrize("key", ["apiKey", "clientSecret", "accessToken", "secretAccessKey", "x-apiKey"]) +def test_is_sensitive_key_splits_camel_case_names(key: str): + assert SensitiveDataMasker().is_sensitive_key(key) is True + + +def test_is_sensitive_key_camel_case_split_keeps_pricing_and_count_fields_clear(): + masker = SensitiveDataMasker() + + assert masker.is_sensitive_key("maxTokens") is False + assert masker.is_sensitive_key("inputCostPerToken") is False + + +def test_empty_non_sensitive_overrides_disables_the_cost_exemption(): + strict = SensitiveDataMasker(non_sensitive_overrides=frozenset()) + + assert SensitiveDataMasker().is_sensitive_key("cost_center_api_key") is False + assert strict.is_sensitive_key("cost_center_api_key") is True + assert strict.is_sensitive_key("cost_center") is False + + +def test_redact_credentials_in_payload_drops_listed_keys_at_any_depth_with_the_given_masker(): + from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload + + payload = { + "logging": [{"callback_vars": {"langfuse_secret_key": "s"}}], + "cost_center_api_key": "sk-cost", + "nested": {"scim_roles": ["r"], "keep": 1, "deeper": [{"logging": {}, "apiKey": "k"}]}, + } + + result = redact_credentials_in_payload( + payload, + SensitiveDataMasker(non_sensitive_overrides=frozenset()), + dropped_keys=frozenset({"logging", "scim_roles"}), + ) + + assert result == { + "cost_center_api_key": "REDACTED", + "nested": {"keep": 1, "deeper": [{"apiKey": "REDACTED"}]}, + } + assert redact_credentials_in_payload(payload)["cost_center_api_key"] == "sk-cost" diff --git a/tests/unit/repositories/test_daily_activity_repository.py b/tests/unit/repositories/test_daily_activity_repository.py index c4021725cd7..e14f49cfbe6 100644 --- a/tests/unit/repositories/test_daily_activity_repository.py +++ b/tests/unit/repositories/test_daily_activity_repository.py @@ -687,11 +687,19 @@ async def test_export_redacts_credential_named_values_inside_metadata_objects() **_export_row("key-1"), "key_metadata": { "cost_center": "cc-1", + "cost_center_api_key": "sk-cost-1234567890", "api_key": "sk-live-1234567890", - "webhook": {"url": "https://hooks.example", "auth_token": "t0k3n"}, + "webhook": {"url": "https://hooks.example", "auth_token": "t0k3n", "clientSecret": "cs"}, "owners": [{"name": "ops", "password": "pw"}], + "scim_roles": ["admin"], + "langfuse": {"apiKey": "lf", "logging": [{"callback_vars": {"langfuse_secret_key": "s"}}]}, + }, + "user_metadata": { + "department": "research", + "github_token": "ghp_secret", + "logging": [{"callback_name": "langfuse"}], + "profile": {"scim_enterprise": {"employeeNumber": "E-1"}, "team": "ops"}, }, - "user_metadata": {"department": "research", "github_token": "ghp_secret"}, }, {**_export_row("key-2"), "key_metadata": {}, "user_metadata": {}}, ), @@ -703,11 +711,17 @@ async def test_export_redacts_credential_named_values_inside_metadata_objects() assert rows[0].key_metadata == { "cost_center": "cc-1", + "cost_center_api_key": "REDACTED", "api_key": "REDACTED", - "webhook": {"url": "https://hooks.example", "auth_token": "REDACTED"}, + "webhook": {"url": "https://hooks.example", "auth_token": "REDACTED", "clientSecret": "REDACTED"}, "owners": [{"name": "ops", "password": "REDACTED"}], + "langfuse": {"apiKey": "REDACTED"}, + } + assert rows[0].user_metadata == { + "department": "research", + "github_token": "REDACTED", + "profile": {"team": "ops"}, } - assert rows[0].user_metadata == {"department": "research", "github_token": "REDACTED"} assert (rows[1].key_metadata, rows[1].user_metadata) == ({}, {}) diff --git a/tests/unit/repositories/test_daily_activity_sql.py b/tests/unit/repositories/test_daily_activity_sql.py index 7257fd207a3..5514129ea55 100644 --- a/tests/unit/repositories/test_daily_activity_sql.py +++ b/tests/unit/repositories/test_daily_activity_sql.py @@ -451,14 +451,14 @@ def test_export_uses_latest_deleted_key_metadata(export_type: ExportType) -> Non assert "COALESCE(vt.user_id, dvt.user_id)" in query.sql +_DROPPED_KEYS: Final = ( + "- 'callback_settings' - 'logging' - 'scim_enterprise' - 'scim_entitlements' - 'scim_roles' " + "- 'secret_manager_settings'" +) _REDACTED_KEY_METADATA: Final = ( - "THEN (COALESCE(vt.metadata, dvt.metadata) - 'callback_settings' - 'logging' - 'secret_manager_settings')" - "::text END), '{}')::jsonb AS key_metadata" -) -_REDACTED_USER_METADATA: Final = ( - "THEN (u.metadata - 'scim_enterprise' - 'scim_entitlements' - 'scim_roles')::text END), '{}')::jsonb " - "AS user_metadata" + f"THEN (COALESCE(vt.metadata, dvt.metadata) {_DROPPED_KEYS})::text END), '{{}}')::jsonb AS key_metadata" ) +_REDACTED_USER_METADATA: Final = f"THEN (u.metadata {_DROPPED_KEYS})::text END), '{{}}')::jsonb AS user_metadata" _EMPTY_KEY_METADATA: Final = "'{}'::jsonb AS key_metadata" _EMPTY_USER_METADATA: Final = "'{}'::jsonb AS user_metadata" From f49ac24c0c0304ee23ae319354fcfe7b09b1db59 Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 3 Nov 2026 23:09:33 +0000 Subject: [PATCH 07/11] fix(masker): keep odd-cased plain names like sEt-CoOkIe sensitive alongside the camelCase split --- litellm/litellm_core_utils/sensitive_data_masker.py | 11 ++++++----- .../litellm_core_utils/test_sensitive_data_masker.py | 5 +++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 2cf8899cf3a..117c96c06e3 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -88,11 +88,12 @@ class SensitiveDataMasker: if excluded_keys and key in excluded_keys: return False - # Split on underscores, hyphens and camelCase boundaries and check if any segment matches the pattern - # This avoids false positives like "max_tokens" matching "token" - # but still catches "api_key", "access_token", "apiKey", "clientSecret", etc. - key_lower: Final = _CAMEL_BOUNDARY.sub("_", str(key)).lower() - key_segments: Final = key_lower.replace("-", "_").split("_") + # Split on underscores and hyphens, and separately on camelCase boundaries, then check whether any + # segment matches the pattern. This avoids false positives like "max_tokens" matching "token" + # but still catches "api_key", "access_token", "apiKey", "clientSecret" and odd-cased "sEt-CoOkIe". + key_lower: Final = str(key).lower().replace("-", "_") + camel_lower: Final = _CAMEL_BOUNDARY.sub("_", str(key)).lower().replace("-", "_") + key_segments: Final = frozenset(key_lower.split("_")) | frozenset(camel_lower.split("_")) # If any segment matches a non-sensitive override, the key is not sensitive. # For example, "input_cost_per_token" contains "token" but also "cost", diff --git a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py index 9039ab99371..b8602954cec 100644 --- a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py @@ -626,6 +626,11 @@ def test_is_sensitive_key_camel_case_split_keeps_pricing_and_count_fields_clear( assert masker.is_sensitive_key("inputCostPerToken") is False +@pytest.mark.parametrize("key", ["sEt-CoOkIe", "AUTHORIZATION", "Api-Key", "X-API-KEY"]) +def test_is_sensitive_key_camel_case_split_does_not_break_odd_cased_plain_names(key: str): + assert SensitiveDataMasker().is_sensitive_key(key) is True + + def test_empty_non_sensitive_overrides_disables_the_cost_exemption(): strict = SensitiveDataMasker(non_sensitive_overrides=frozenset()) From 9160e4b8c4031beabc3b7accf3fe92f1491bd06c Mon Sep 17 00:00:00 2001 From: jesus Date: Tue, 3 Nov 2026 23:25:47 +0000 Subject: [PATCH 08/11] test(masker): mark new test locals Final --- .../unit/litellm_core_utils/test_sensitive_data_masker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py index b8602954cec..9bb7d7ad3fe 100644 --- a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py @@ -620,7 +620,7 @@ def test_is_sensitive_key_splits_camel_case_names(key: str): def test_is_sensitive_key_camel_case_split_keeps_pricing_and_count_fields_clear(): - masker = SensitiveDataMasker() + masker: Final = SensitiveDataMasker() assert masker.is_sensitive_key("maxTokens") is False assert masker.is_sensitive_key("inputCostPerToken") is False @@ -632,7 +632,7 @@ def test_is_sensitive_key_camel_case_split_does_not_break_odd_cased_plain_names( def test_empty_non_sensitive_overrides_disables_the_cost_exemption(): - strict = SensitiveDataMasker(non_sensitive_overrides=frozenset()) + strict: Final = SensitiveDataMasker(non_sensitive_overrides=frozenset()) assert SensitiveDataMasker().is_sensitive_key("cost_center_api_key") is False assert strict.is_sensitive_key("cost_center_api_key") is True @@ -642,13 +642,13 @@ def test_empty_non_sensitive_overrides_disables_the_cost_exemption(): def test_redact_credentials_in_payload_drops_listed_keys_at_any_depth_with_the_given_masker(): from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload - payload = { + payload: Final = { "logging": [{"callback_vars": {"langfuse_secret_key": "s"}}], "cost_center_api_key": "sk-cost", "nested": {"scim_roles": ["r"], "keep": 1, "deeper": [{"logging": {}, "apiKey": "k"}]}, } - result = redact_credentials_in_payload( + result: Final = redact_credentials_in_payload( payload, SensitiveDataMasker(non_sensitive_overrides=frozenset()), dropped_keys=frozenset({"logging", "scim_roles"}), From 4c4fe6e29600c4783b0a8fda8c87ebc79752a377 Mon Sep 17 00:00:00 2001 From: jesus Date: Fri, 6 Nov 2026 01:10:15 +0000 Subject: [PATCH 09/11] fix(masker): make the camelCase split opt-in and enable it only for the usage export masker --- .../sensitive_data_masker.py | 10 +++++++--- .../repositories/daily_activity_repository.py | 2 +- .../test_sensitive_data_masker.py | 19 ++++++++++++------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 117c96c06e3..62958a2d8a7 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -48,6 +48,7 @@ class SensitiveDataMasker: mask_char: str = "*", mask_short_values: bool = True, extra_sensitive_patterns: AbstractSet[str] | None = None, + split_camel_case: bool = False, ): self.sensitive_patterns = (sensitive_patterns or _DEFAULT_SENSITIVE_PATTERNS) | ( extra_sensitive_patterns or frozenset() @@ -63,6 +64,7 @@ class SensitiveDataMasker: self.visible_suffix = visible_suffix self.mask_char = mask_char self.mask_short_values = mask_short_values + self.split_camel_case = split_camel_case def mask_value(self, value: str) -> str: value_str: Final = str(value) @@ -88,11 +90,13 @@ class SensitiveDataMasker: if excluded_keys and key in excluded_keys: return False - # Split on underscores and hyphens, and separately on camelCase boundaries, then check whether any + # Split on underscores and hyphens (and, when opted in, camelCase boundaries too) and check whether any # segment matches the pattern. This avoids false positives like "max_tokens" matching "token" - # but still catches "api_key", "access_token", "apiKey", "clientSecret" and odd-cased "sEt-CoOkIe". + # but still catches "api_key", "access_token" and, with split_camel_case, "apiKey" and "clientSecret". key_lower: Final = str(key).lower().replace("-", "_") - camel_lower: Final = _CAMEL_BOUNDARY.sub("_", str(key)).lower().replace("-", "_") + camel_lower: Final = ( + _CAMEL_BOUNDARY.sub("_", str(key)).lower().replace("-", "_") if self.split_camel_case else key_lower + ) key_segments: Final = frozenset(key_lower.split("_")) | frozenset(camel_lower.split("_")) # If any segment matches a non-sensitive override, the key is not sensitive. diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index ae4d9992036..bafd69172b8 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -143,7 +143,7 @@ def _daily_rows_table( raise AssertionError("unreachable") -_EXPORT_METADATA_MASKER: Final = SensitiveDataMasker(non_sensitive_overrides=frozenset()) +_EXPORT_METADATA_MASKER: Final = SensitiveDataMasker(non_sensitive_overrides=frozenset(), split_camel_case=True) def _redacted_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py index 9bb7d7ad3fe..13b61e4cebf 100644 --- a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py @@ -615,20 +615,25 @@ def test_redact_credentials_in_payload_leaves_a_realistic_tool_schema_intact(): @pytest.mark.parametrize("key", ["apiKey", "clientSecret", "accessToken", "secretAccessKey", "x-apiKey"]) -def test_is_sensitive_key_splits_camel_case_names(key: str): - assert SensitiveDataMasker().is_sensitive_key(key) is True +def test_split_camel_case_opt_in_catches_camel_case_credential_names(key: str): + assert SensitiveDataMasker(split_camel_case=True).is_sensitive_key(key) is True -def test_is_sensitive_key_camel_case_split_keeps_pricing_and_count_fields_clear(): - masker: Final = SensitiveDataMasker() +@pytest.mark.parametrize("key", ["apiKey", "clientSecret", "tokenCount", "keyAlias", "accessLevel"]) +def test_default_masker_does_not_split_camel_case(key: str): + assert SensitiveDataMasker().is_sensitive_key(key) is False + + +def test_split_camel_case_keeps_pricing_and_count_fields_clear(): + masker: Final = SensitiveDataMasker(split_camel_case=True) assert masker.is_sensitive_key("maxTokens") is False assert masker.is_sensitive_key("inputCostPerToken") is False @pytest.mark.parametrize("key", ["sEt-CoOkIe", "AUTHORIZATION", "Api-Key", "X-API-KEY"]) -def test_is_sensitive_key_camel_case_split_does_not_break_odd_cased_plain_names(key: str): - assert SensitiveDataMasker().is_sensitive_key(key) is True +def test_split_camel_case_does_not_break_odd_cased_plain_names(key: str): + assert SensitiveDataMasker(split_camel_case=True).is_sensitive_key(key) is True def test_empty_non_sensitive_overrides_disables_the_cost_exemption(): @@ -650,7 +655,7 @@ def test_redact_credentials_in_payload_drops_listed_keys_at_any_depth_with_the_g result: Final = redact_credentials_in_payload( payload, - SensitiveDataMasker(non_sensitive_overrides=frozenset()), + SensitiveDataMasker(non_sensitive_overrides=frozenset(), split_camel_case=True), dropped_keys=frozenset({"logging", "scim_roles"}), ) From b3bab0065f83c3ddeb8fc64bc1882aca4c151041 Mon Sep 17 00:00:00 2001 From: jesus Date: Fri, 6 Nov 2026 03:43:06 +0000 Subject: [PATCH 10/11] fix(usage-export): mask acronym-cased and unseparated credential names in exported metadata --- .../sensitive_data_masker.py | 4 +-- .../repositories/daily_activity_repository.py | 26 ++++++++++++++++++- .../test_sensitive_data_masker.py | 15 ++++++++++- .../test_daily_activity_repository.py | 8 ++++++ 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 62958a2d8a7..2e6d54d19d8 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -31,7 +31,7 @@ _DEFAULT_SENSITIVE_PATTERNS: Final = frozenset( ) -_CAMEL_BOUNDARY: Final = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") +_CAMEL_BOUNDARY: Final = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])") def _object_items(data: Mapping[str, object]) -> ItemsView[str, object]: @@ -92,7 +92,7 @@ class SensitiveDataMasker: # Split on underscores and hyphens (and, when opted in, camelCase boundaries too) and check whether any # segment matches the pattern. This avoids false positives like "max_tokens" matching "token" - # but still catches "api_key", "access_token" and, with split_camel_case, "apiKey" and "clientSecret". + # but still catches "api_key", "access_token" and, with split_camel_case, "apiKey", "APIKey" and "AWSSecret". key_lower: Final = str(key).lower().replace("-", "_") camel_lower: Final = ( _CAMEL_BOUNDARY.sub("_", str(key)).lower().replace("-", "_") if self.split_camel_case else key_lower diff --git a/litellm/repositories/daily_activity_repository.py b/litellm/repositories/daily_activity_repository.py index bafd69172b8..a66d86da1d5 100644 --- a/litellm/repositories/daily_activity_repository.py +++ b/litellm/repositories/daily_activity_repository.py @@ -143,7 +143,31 @@ def _daily_rows_table( raise AssertionError("unreachable") -_EXPORT_METADATA_MASKER: Final = SensitiveDataMasker(non_sensitive_overrides=frozenset(), split_camel_case=True) +_EXPORT_UNSEPARATED_CREDENTIAL_NAMES: Final[frozenset[str]] = frozenset( + { + "apikey", + "apitoken", + "accesskey", + "accesstoken", + "authkey", + "authtoken", + "bearertoken", + "clientsecret", + "passwd", + "privatekey", + "pwd", + "refreshtoken", + "secretkey", + "sessiontoken", + "signingkey", + "webhooksecret", + } +) +_EXPORT_METADATA_MASKER: Final = SensitiveDataMasker( + non_sensitive_overrides=frozenset(), + extra_sensitive_patterns=_EXPORT_UNSEPARATED_CREDENTIAL_NAMES, + split_camel_case=True, +) def _redacted_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py index 13b61e4cebf..34934a0bca1 100644 --- a/tests/unit/litellm_core_utils/test_sensitive_data_masker.py +++ b/tests/unit/litellm_core_utils/test_sensitive_data_masker.py @@ -614,7 +614,20 @@ def test_redact_credentials_in_payload_leaves_a_realistic_tool_schema_intact(): assert result["api_key"] == "REDACTED" -@pytest.mark.parametrize("key", ["apiKey", "clientSecret", "accessToken", "secretAccessKey", "x-apiKey"]) +@pytest.mark.parametrize( + "key", + [ + "apiKey", + "clientSecret", + "accessToken", + "secretAccessKey", + "x-apiKey", + "APIKey", + "OpenAIAPIKey", + "AWSSecret", + "XAPIKey", + ], +) def test_split_camel_case_opt_in_catches_camel_case_credential_names(key: str): assert SensitiveDataMasker(split_camel_case=True).is_sensitive_key(key) is True diff --git a/tests/unit/repositories/test_daily_activity_repository.py b/tests/unit/repositories/test_daily_activity_repository.py index e14f49cfbe6..b554c72af0a 100644 --- a/tests/unit/repositories/test_daily_activity_repository.py +++ b/tests/unit/repositories/test_daily_activity_repository.py @@ -693,6 +693,10 @@ async def test_export_redacts_credential_named_values_inside_metadata_objects() "owners": [{"name": "ops", "password": "pw"}], "scim_roles": ["admin"], "langfuse": {"apiKey": "lf", "logging": [{"callback_vars": {"langfuse_secret_key": "s"}}]}, + "OpenAIAPIKey": "sk-acronym", + "apikey": "sk-unseparated", + "APIKEY": "sk-upper", + "passwd": "pw-short", }, "user_metadata": { "department": "research", @@ -716,6 +720,10 @@ async def test_export_redacts_credential_named_values_inside_metadata_objects() "webhook": {"url": "https://hooks.example", "auth_token": "REDACTED", "clientSecret": "REDACTED"}, "owners": [{"name": "ops", "password": "REDACTED"}], "langfuse": {"apiKey": "REDACTED"}, + "OpenAIAPIKey": "REDACTED", + "apikey": "REDACTED", + "APIKEY": "REDACTED", + "passwd": "REDACTED", } assert rows[0].user_metadata == { "department": "research", From defd5720d18c1653153cd72cd60c08d4c86206f7 Mon Sep 17 00:00:00 2001 From: jesus Date: Fri, 6 Nov 2026 03:47:55 +0000 Subject: [PATCH 11/11] test(e2e): assert acronym-cased and upper-case credential names are masked in the usage export --- tests/e2e/models.py | 1 + .../spend_tracking/test_usage_export_metadata_e2e.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 776faf33966..01721396922 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -61,6 +61,7 @@ class KeyMetadata(BaseModel): guardrails: list[str] | None = None webhook_token: str | None = None apiKey: str | None = None + OpenAIAPIKey: str | None = None class ObjectPermission(BaseModel): diff --git a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py index 6c908915034..42d460daf8f 100644 --- a/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_usage_export_metadata_e2e.py @@ -48,12 +48,14 @@ class UsageExportKeyMetadata(BaseModel, frozen=True): tag: str | None = None webhook_token: str | None = None apiKey: str | None = None + OpenAIAPIKey: str | None = None class UsageExportUserMetadata(BaseModel, frozen=True): department: str | None = None ldap_password: str | None = None clientSecret: str | None = None + APIKEY: str | None = None class UsageExportRow(BaseModel, frozen=True): @@ -113,6 +115,7 @@ class TestUsageExportMetadata: "department": f"research-{marker}", "ldap_password": f"pw-{marker}", "clientSecret": f"cs-{marker}", + "APIKEY": f"upper-{marker}", }, ), response_type=UserNewResponse, @@ -125,7 +128,10 @@ class TestUsageExportMetadata: models=[model], key_alias=f"e2e-usage-export-{marker}", metadata=KeyMetadata( - tag=f"cost-center-{marker}", webhook_token=f"whsec-{marker}", apiKey=f"ak-{marker}" + tag=f"cost-center-{marker}", + webhook_token=f"whsec-{marker}", + apiKey=f"ak-{marker}", + OpenAIAPIKey=f"acr-{marker}", ), ) ) @@ -171,10 +177,12 @@ class TestUsageExportMetadata: masked = ( row.key_metadata.webhook_token, row.key_metadata.apiKey, + row.key_metadata.OpenAIAPIKey, row.user_metadata.ldap_password, row.user_metadata.clientSecret, + row.user_metadata.APIKEY, ) - assert masked == ("REDACTED",) * 4, f"credential-named metadata values reached the export unmasked: {row}" + assert masked == ("REDACTED",) * 6, f"credential-named metadata values reached the export unmasked: {row}" own_view: Final = client.proxy.transport.probe( EXPORT_ROUTE,