From e40037f4c4df7149fa0bddc73eeb45f44f68ee78 Mon Sep 17 00:00:00 2001 From: mrinal Date: Mon, 5 Oct 2026 21:26:04 +0000 Subject: [PATCH] feat(spend_logs): configure which metadata fields are stored in LiteLLM_SpendLogs Adds general_settings.spend_logs_metadata_fields with mutually exclusive include and exclude lists. The filter runs on a copy of the row right before it is queued for Postgres, so daily spend rollups, budgets and callbacks still see every key. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 40 +++++ litellm/proxy/db/db_spend_update_writer.py | 18 ++- litellm/proxy/proxy_server.py | 5 + .../spend_tracking/spend_tracking_utils.py | 34 ++++- .../spend/test_spend_logs_metadata_fields.py | 109 ++++++++++++++ .../test_spend_tracking_utils.py | 142 +++++++++++++++--- 6 files changed, 321 insertions(+), 27 deletions(-) create mode 100644 tests/integration/spend/test_spend_logs_metadata_fields.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b993a5d8d6d..be1b8e6e6c8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple, Ty import httpx from pydantic import ( + AfterValidator, BaseModel, BeforeValidator, ConfigDict, @@ -2721,6 +2722,41 @@ class ScheduledJobStaggerSettings(LiteLLMPydanticObjectBase): ) +SPEND_LOGS_METADATA_ALWAYS_KEPT_FIELDS: Final = frozenset({"status", "cold_storage_object_key"}) + + +def _known_spend_logs_metadata_field(name: str) -> str: + if name not in SpendLogsMetadata.__annotations__: + raise ValueError(f"{name!r} is not a LiteLLM_SpendLogs.metadata field") + return name + + +SpendLogsMetadataFieldName: TypeAlias = Annotated[str, AfterValidator(_known_spend_logs_metadata_field)] + + +class SpendLogsMetadataFields(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + include: tuple[SpendLogsMetadataFieldName, ...] | None = None + exclude: tuple[SpendLogsMetadataFieldName, ...] | None = None + + @model_validator(mode="after") + def _exactly_one_list(self) -> "SpendLogsMetadataFields": + if (self.include is None) == (self.exclude is None): + raise ValueError("set exactly one of 'include' or 'exclude'") + always_kept_excluded: Final = sorted(SPEND_LOGS_METADATA_ALWAYS_KEPT_FIELDS.intersection(self.exclude or ())) + if always_kept_excluded: + raise ValueError(f"{always_kept_excluded} are always kept and cannot be excluded") + return self + + def keeps(self, name: str) -> bool: + if name in SPEND_LOGS_METADATA_ALWAYS_KEPT_FIELDS: + return True + if self.include is not None: + return name in self.include + return name not in (self.exclude or ()) + + class ConfigGeneralSettings(LiteLLMPydanticObjectBase): """ Documents all the fields supported by `general_settings` in config.yaml @@ -3037,6 +3073,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, stores request messages and responses in spend logs. Default is False.", ) + spend_logs_metadata_fields: SpendLogsMetadataFields | None = Field( + None, + description="Which keys of LiteLLM_SpendLogs.metadata are written to the database. Set exactly one of 'include' (write only these keys) or 'exclude' (drop these keys). 'status' and 'cold_storage_object_key' are always written. Daily spend tables, budgets and logging callbacks still see every key. Unset writes every key", + ) disable_auto_add_proxy_admin_to_teams: bool | None = Field( None, description="By default, the user calling /team/new is automatically added to the new team as a team admin. If True, proxy admins are no longer auto-added; members explicitly listed in members_with_roles are unaffected. Default is False.", diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 5fdde118fb9..b4e85b7b82d 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -652,7 +652,14 @@ class DBSpendUpdateWriter: from litellm.repositories.table_repositories import SpendLogsRepository request_id: Final = payload["request_id"] - row: Final = _batch_cost_row_to_write(payload, disable_spend_logs) + from litellm.proxy.spend_tracking.spend_tracking_utils import ( + configured_spend_logs_metadata_fields, + spend_log_row_with_retained_metadata, + ) + + row: Final = spend_log_row_with_retained_metadata( + _batch_cost_row_to_write(payload, disable_spend_logs), configured_spend_logs_metadata_fields() + ) spend_logs: Final = SpendLogsRepository(prisma_client).table try: claimed: Final = await spend_logs.create_many( @@ -1511,9 +1518,16 @@ class DBSpendUpdateWriter: ) ) if prisma_client is not None and spend_logs_url is not None or prisma_client is not None: + from litellm.proxy.spend_tracking.spend_tracking_utils import ( + configured_spend_logs_metadata_fields, + spend_log_row_with_retained_metadata, + ) from litellm.proxy.utils import enqueue_spend_logs, request_spend_log_flush - await enqueue_spend_logs(prisma_client, (payload,)) + await enqueue_spend_logs( + prisma_client, + (spend_log_row_with_retained_metadata(payload, configured_spend_logs_metadata_fields()),), + ) if payload.get("call_type") in RESPONSES_SESSION_CALL_TYPES: request_spend_log_flush(prisma_client) else: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6d1a04c2b5e..955ecebc57a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6782,6 +6782,11 @@ class ProxyConfig: ).user_api_key_cache_max_size ) + if "spend_logs_metadata_fields" in general_settings: + _ = ConfigGeneralSettings.model_validate( + MappingProxyType({"spend_logs_metadata_fields": general_settings["spend_logs_metadata_fields"]}) + ) + ### PKCE MULTI-INSTANCE PREREQUISITE CHECK ### # PKCE verifiers are stored in redis_usage_cache when available so they can # be read back by any instance (not just the one that started the auth flow). diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index b71b834a31c..ada16baa6c9 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -9,7 +9,7 @@ from functools import reduce from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable -from pydantic import BaseModel, JsonValue +from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -47,7 +47,12 @@ from litellm.litellm_core_utils.ptu_pricing import azure_spillover from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker from litellm.llms.anthropic.common_utils import resolve_used_client_oauth_token -from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload, SpendLogsRouterMetadata +from litellm.proxy._types import ( + SpendLogsMetadata, + SpendLogsMetadataFields, + SpendLogsPayload, + SpendLogsRouterMetadata, +) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.proxy.utils import PrismaClient, hash_token @@ -1720,6 +1725,31 @@ def should_store_prompts_and_responses_in_spend_logs() -> bool: return get_secret_bool("STORE_PROMPTS_IN_SPEND_LOGS") is True +_SPEND_LOGS_METADATA_FIELDS_ADAPTER: Final = TypeAdapter(SpendLogsMetadataFields | None) +_SPEND_LOGS_METADATA_ADAPTER: Final = TypeAdapter(dict[str, JsonValue]) + + +def configured_spend_logs_metadata_fields() -> SpendLogsMetadataFields | None: + from litellm.proxy.proxy_server import general_settings + + try: + return _SPEND_LOGS_METADATA_FIELDS_ADAPTER.validate_python(general_settings.get("spend_logs_metadata_fields")) + except ValidationError as e: + verbose_proxy_logger.error("Ignoring invalid general_settings.spend_logs_metadata_fields: %s", e) + return None + + +def spend_log_row_with_retained_metadata( + row: Mapping[str, object], fields: SpendLogsMetadataFields | None +) -> Mapping[str, object]: + metadata_json: Final = row.get("metadata") + if fields is None or not isinstance(metadata_json, str): + return row + metadata: Final = _SPEND_LOGS_METADATA_ADAPTER.validate_json(metadata_json) + retained: Final = {name: value for name, value in metadata.items() if fields.keeps(name)} + return {**row, "metadata": safe_dumps(retained)} + + def _get_status_for_spend_log( metadata: dict, ) -> Literal["success", "failure"]: diff --git a/tests/integration/spend/test_spend_logs_metadata_fields.py b/tests/integration/spend/test_spend_logs_metadata_fields.py new file mode 100644 index 00000000000..c9128c18ce5 --- /dev/null +++ b/tests/integration/spend/test_spend_logs_metadata_fields.py @@ -0,0 +1,109 @@ +import json +from collections.abc import Mapping +from hashlib import sha256 +from pathlib import Path +from typing import Final +from uuid import uuid4 + +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +CACHED_PROMPT_TOKENS: Final = 4 + + +def _config(tmp_path: Path, spend_logs_metadata_fields: Mapping[str, JsonValue]) -> Path: + config: Final = tmp_path / f"spend-logs-metadata-fields-{uuid4()}.json" + config.write_text( + json.dumps( + { + "model_list": [], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "store_model_in_db": True, + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + "spend_logs_metadata_fields": dict(spend_logs_metadata_fields), + }, + } + ) + ) + return config + + +def _respond(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + return Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid4()}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 2, + "total_tokens": 12, + "prompt_tokens_details": {"cached_tokens": CACHED_PROMPT_TOKENS}, + }, + } + ).encode() + ) + + +def _stored_row_after_one_chat( + gateway: Gateway, tmp_path: Path, spend_logs_metadata_fields: Mapping[str, JsonValue] +) -> tuple[dict[str, JsonValue], str]: + with ( + wire_server(_respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_config(tmp_path, spend_logs_metadata_fields)) as isolated, + isolated.scenario() as scenario, + ): + model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key") + api_key: Final = scenario.key(key_alias=f"metadata-fields-{uuid4()}", models=[model]) + response_id: Final = string_value(isolated.chat(model, key=api_key)["id"]) + rows: Final = eventually( + lambda: read_rows( + 'SELECT metadata, proxy_server_request, response FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (response_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + return rows[0], sha256(api_key.encode()).hexdigest() + + +def test_excluded_metadata_fields_are_not_stored_but_still_reach_daily_spend(gateway: Gateway, tmp_path: Path) -> None: + row, hashed_key = _stored_row_after_one_chat( + gateway, tmp_path, {"exclude": ["model_map_information", "usage_object"]} + ) + + metadata: Final = object_value(row["metadata"]) + assert "model_map_information" not in metadata, metadata + assert "usage_object" not in metadata, metadata + assert {"status", "cold_storage_object_key"} <= set(metadata), metadata + assert string_value(metadata["user_api_key_alias"]).startswith("metadata-fields-") + assert row["proxy_server_request"] == {} + assert row["response"] == {} + daily: Final = eventually( + lambda: read_rows( + 'SELECT prompt_tokens, cache_read_input_tokens FROM "LiteLLM_DailyUserSpend" WHERE api_key=%s', + (hashed_key,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert daily[0]["cache_read_input_tokens"] == CACHED_PROMPT_TOKENS, daily + + +def test_included_metadata_fields_are_the_only_ones_stored_besides_always_kept( + gateway: Gateway, tmp_path: Path +) -> None: + row, _ = _stored_row_after_one_chat(gateway, tmp_path, {"include": ["user_api_key_alias"]}) + + assert set(object_value(row["metadata"])) == {"status", "cold_storage_object_key", "user_api_key_alias"} diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index a3de9328437..279c82c5224 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -23,7 +23,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth +from litellm.proxy._types import SpendLogsMetadataFields, SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_tracking_utils import ( @@ -41,9 +41,11 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_guardrail_information_for_spend_logs, _sanitize_request_body_for_spend_logs_payload, _scrub_raw_model_from_error_information, + configured_spend_logs_metadata_fields, get_logging_payload, get_spend_logs_id, should_store_prompts_and_responses_in_spend_logs, + spend_log_row_with_retained_metadata, ) from litellm.proxy.utils import hash_token from litellm.types.router import GenericLiteLLMParams @@ -83,10 +85,16 @@ def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch "classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]}, "originating_request_masked": {"input": "source-only", "api_key": "REDACTED"}, } - stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload( - metadata={}, litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}}, - kwargs={"standard_logging_object": audit, "standard_callback_dynamic_params": {"turn_off_message_logging": redact}}, - )) + stored: Final = json.loads( + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, + litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}}, + kwargs={ + "standard_logging_object": audit, + "standard_callback_dynamic_params": {"turn_off_message_logging": redact}, + }, + ) + ) if not store_prompts or redact: assert "classifier_input" not in stored assert "originating_request_masked" not in stored @@ -212,9 +220,7 @@ def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id(): from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1") - cost_session: Final = _get_batch_trace_session_id( - call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost" - ) + cost_session: Final = _get_batch_trace_session_id(call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost") assert create_session == cost_session == "batch-uid-1" @@ -2756,7 +2762,11 @@ def test_proxy_server_request_payload_redacts_provider_credentials(mock_should_s "extra_headers": {"Authorization": "Bearer canary-extra-header"}, "tools": [ {"type": "function", "function": {"name": "f", "parameters": tool_parameters}}, - {"type": "mcp", "server_url": "https://mcp.example.com", "headers": {"Authorization": "canary-mcp"}}, + { + "type": "mcp", + "server_url": "https://mcp.example.com", + "headers": {"Authorization": "canary-mcp"}, + }, ], "fallbacks": [{"model": "azure-b", **credentials}], "metadata": metadata, @@ -5310,7 +5320,7 @@ ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = ( 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' '"usage":{"output_tokens":4}}\n\n', - "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', ) @@ -5348,9 +5358,7 @@ def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_ """ logging_obj = _anthropic_messages_logging_obj(stream=False) - logged_response = logging_obj._handle_anthropic_messages_response_logging( - result=ANTHROPIC_MESSAGES_RESPONSE - ) + logged_response = logging_obj._handle_anthropic_messages_response_logging(result=ANTHROPIC_MESSAGES_RESPONSE) assert logged_response.id == "msg_01Lit6806NonStreaming" assert ( @@ -5426,9 +5434,7 @@ def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_prov end_time=datetime.datetime.now(timezone.utc), logging_obj=logging_obj, ) - assert logging_obj.model_call_details["complete_streaming_response"].id == ( - "6806cafe-0000-4000-8000-000000000001" - ) + assert logging_obj.model_call_details["complete_streaming_response"].id == ("6806cafe-0000-4000-8000-000000000001") def test_spend_log_request_id_for_chat_completions_is_untouched(): @@ -5510,6 +5516,7 @@ def test_failed_agent_request_keeps_registered_display_name(): assert payload["status"] == "failure" assert payload["model_id"] == "registered-agent" + _CLI_SESSION_ALIAS: Final = "cli-session-alice" _CLI_SESSION_TOKEN: Final = "cli-session-Qm7xJ2kP9sLw4vT1nR8yAa" @@ -5641,15 +5648,104 @@ def test_baseline_estimate_metadata_comes_from_the_logging_stamp() -> None: def test_untrusted_agent_label_cannot_replace_verified_billing_identity(billing_agent: str | None) -> None: kwargs = { "model": "gpt-4", - "litellm_params": {"metadata": { - "user_api_key": "test-key", - "agent_id": "header-selected-agent", - "billing_agent_id": billing_agent, - }}, + "litellm_params": { + "metadata": { + "user_api_key": "test-key", + "agent_id": "header-selected-agent", + "billing_agent_id": billing_agent, + } + }, } payload = get_logging_payload( - kwargs=kwargs, response_obj={"id": "request"}, - start_time=datetime.datetime.now(timezone.utc), end_time=datetime.datetime.now(timezone.utc), + kwargs=kwargs, + response_obj={"id": "request"}, + start_time=datetime.datetime.now(timezone.utc), + end_time=datetime.datetime.now(timezone.utc), ) assert payload["agent_id"] == "header-selected-agent" assert payload["billing_agent_id"] == billing_agent + + +def _spend_log_row(metadata: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType({"request_id": "req-1", "spend": 0.5, "metadata": json.dumps(dict(metadata))}) + + +_STORED_METADATA: Final = MappingProxyType( + { + "status": "success", + "cold_storage_object_key": "logs/req-1.json", + "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"max_tokens": 10}}, + "usage_object": {"prompt_tokens": 3}, + "user_api_key_alias": "alias", + } +) + + +def test_spend_log_row_keeps_every_metadata_field_when_unconfigured() -> None: + row: Final = _spend_log_row(_STORED_METADATA) + + assert spend_log_row_with_retained_metadata(row, None) is row + + +def test_spend_log_row_drops_excluded_metadata_fields_only() -> None: + row: Final = _spend_log_row(_STORED_METADATA) + + stored: Final = spend_log_row_with_retained_metadata( + row, SpendLogsMetadataFields(exclude=("model_map_information", "user_api_key_alias")) + ) + + assert json.loads(cast(str, stored["metadata"])) == { + "status": "success", + "cold_storage_object_key": "logs/req-1.json", + "usage_object": {"prompt_tokens": 3}, + } + assert {name: value for name, value in stored.items() if name != "metadata"} == { + "request_id": "req-1", + "spend": 0.5, + } + + +def test_spend_log_row_include_keeps_listed_and_always_kept_fields() -> None: + stored: Final = spend_log_row_with_retained_metadata( + _spend_log_row(_STORED_METADATA), SpendLogsMetadataFields(include=("usage_object",)) + ) + + assert json.loads(cast(str, stored["metadata"])) == { + "status": "success", + "cold_storage_object_key": "logs/req-1.json", + "usage_object": {"prompt_tokens": 3}, + } + + +@pytest.mark.parametrize( + "configured", + [ + {"include": ["usage_object"], "exclude": ["model_map_information"]}, + {}, + {"exclude": ["model_map_informaton"]}, + {"include": ["usage_object", "not_a_field"]}, + {"exclude": ["status"]}, + {"exclude": ["cold_storage_object_key"]}, + {"exclude": ["model_map_information"], "drop": ["usage_object"]}, + ], +) +def test_spend_logs_metadata_fields_rejects_ambiguous_or_lossy_config(configured: dict[str, list[str]]) -> None: + from pydantic import ValidationError + + from litellm.proxy._types import ConfigGeneralSettings + + with pytest.raises(ValidationError): + ConfigGeneralSettings.model_validate({"spend_logs_metadata_fields": configured}) + + +def test_configured_spend_logs_metadata_fields_ignores_invalid_runtime_value() -> None: + with patch( + "litellm.proxy.proxy_server.general_settings", + {"spend_logs_metadata_fields": {"include": ["usage_object"], "exclude": ["status"]}}, + ): + assert configured_spend_logs_metadata_fields() is None + with patch( + "litellm.proxy.proxy_server.general_settings", + {"spend_logs_metadata_fields": {"exclude": ["model_map_information"]}}, + ): + assert configured_spend_logs_metadata_fields() == SpendLogsMetadataFields(exclude=("model_map_information",))