diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index dd873ae49dd..96942b2a260 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2833,6 +2833,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, stores request messages and responses in spend logs. Default is False.", ) + spend_logs_prompt_storage: Literal["latest_user_message"] | None = Field( + None, + description="With store_prompts_in_spend_logs on, 'latest_user_message' stores only the text of the request's latest user message and no response. Unset stores the full request and response.", + ) claude_code_marketplace_auth_required: bool = Field( False, description="If True, GET /claude-code/marketplace.json requires a LiteLLM key in the Authorization or x-litellm-api-key header (?key= is not accepted) and answers 401 with WWW-Authenticate: Bearer without one, and GET /public/skill_hub returns an empty list. Default is False, which serves the enabled plugins to anyone.", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 94b079163e9..b990dffd230 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -58,6 +58,7 @@ from litellm.litellm_core_utils.litellm_logging import ( is_valid_sha256_hash, request_model_access_groups_from_litellm_params, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.litellm_core_utils.ptu_pricing import azure_spillover from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker @@ -1198,7 +1199,7 @@ def _get_messages_for_spend_logs_payload( metadata: dict | None = None, ) -> str: if should_store_prompts_and_responses_in_spend_logs(): - if standard_logging_payload is not None: + if standard_logging_payload is not None and not stores_latest_user_message_only(): call_type: Final = standard_logging_payload.get("call_type", "") if call_type == "_arealtime": messages: Final = standard_logging_payload.get("messages") @@ -1214,6 +1215,77 @@ _SENSITIVE_REQUEST_BODY_KEYS: Final = frozenset({"secret_fields"}) _REQUEST_BODY_CREDENTIAL_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns=frozenset({"apikey"})) _TOOL_INPUT_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use", "mcp_tool_use"}) _TOOL_OUTPUT_BLOCK_TYPES: Final = frozenset({"tool_result", "mcp_tool_result", "function_call_output"}) +_LATEST_USER_MESSAGE_STORAGE: Final = "latest_user_message" +_TEXT_BLOCK_TYPES: Final = frozenset({"text", "input_text"}) +_OMITTED_BLOCK_PLACEHOLDERS: Final = MappingProxyType( + { + "image": "[image omitted]", + "image_url": "[image omitted]", + "input_image": "[image omitted]", + "file": "[file omitted]", + "input_file": "[file omitted]", + "document": "[file omitted]", + "input_audio": "[audio omitted]", + } +) + + +def stores_latest_user_message_only() -> bool: + from litellm.proxy.proxy_server import general_settings_view + + return general_settings_view().get("spend_logs_prompt_storage") == _LATEST_USER_MESSAGE_STORAGE + + +def stores_full_prompts_in_spend_logs() -> bool: + return should_store_prompts_and_responses_in_spend_logs() and not stores_latest_user_message_only() + + +def _block_type(block: object) -> str | None: + block_type: Final = message_field(block, "type") + return block_type if isinstance(block_type, str) else None + + +def _content_block_text(block: object) -> str | None: + if isinstance(block, str): + return block + block_type: Final = _block_type(block) + if block_type in _TEXT_BLOCK_TYPES: + text: Final = message_field(block, "text") + return text if isinstance(text, str) else None + return _OMITTED_BLOCK_PLACEHOLDERS.get(block_type) if block_type is not None else None + + +def _is_tool_output_block(block: object) -> bool: + return _block_type(block) in _TOOL_OUTPUT_BLOCK_TYPES + + +def _user_content_text(content: object) -> str | None: + if isinstance(content, str): + return content or None + blocks: Final = parts_of(content) + if any(_is_tool_output_block(block) for block in blocks): + return None + texts: Final = tuple(text for text in map(_content_block_text, blocks) if text) + return "\n".join(texts) or None + + +def _latest_user_message_text(request_body: Mapping[str, object]) -> str | None: + turns: Final = request_body.get("messages", request_body.get("input")) + if isinstance(turns, str): + return turns or None + latest: Final = next(reversed(parts_of(turns)), None) + if message_field(latest, "role") != "user": + return None + return _user_content_text(message_field(latest, "content")) + + +def _latest_user_message_body(request_body: Mapping[str, object]) -> Mapping[str, object]: + text: Final = _latest_user_message_text(request_body) + return MappingProxyType({}) if text is None else MappingProxyType({_LATEST_USER_MESSAGE_STORAGE: text}) + + +def _stored_request_body(request_body: Mapping[str, object]) -> Mapping[str, object]: + return _latest_user_message_body(request_body) if stores_latest_user_message_only() else request_body def _is_request_body_credential(key: str, value: object) -> bool: @@ -1476,7 +1548,7 @@ def _sanitize_guardrail_information_for_spend_logs( here to match OTEL's defensive read pattern; otherwise iteration would yield the dict's keys and crash the whole spend-log write. """ - if guardrail_information is None or should_store_prompts_and_responses_in_spend_logs(): + if guardrail_information is None or stores_full_prompts_in_spend_logs(): return guardrail_information entries: Final = [guardrail_information] if isinstance(guardrail_information, dict) else guardrail_information return [_redact_prompt_fields_in_guardrail_entry(entry) for entry in entries if isinstance(entry, dict)] @@ -1554,7 +1626,7 @@ def sanitize_error_information_for_spend_logs( ) sanitized = cast(dict, {**persisted}) - if not should_store_prompts_and_responses_in_spend_logs(): + if not stores_full_prompts_in_spend_logs(): for field in ("error_message", "traceback"): value = sanitized.get(field) if isinstance(value, str): @@ -1895,7 +1967,7 @@ def _get_proxy_server_request_for_spend_logs_payload( _request_body = _convert_mapping_to_json_serializable(without_classifier_audit(_request_body)) perform_redaction(model_call_details=_request_body, result=None) - stored_request_body: Final = _sanitize_proxy_server_request_body(_request_body) + stored_request_body: Final = _sanitize_proxy_server_request_body(_stored_request_body(_request_body)) _request_body_json_str: Final = safe_dumps( _stored_request_without_requester_ip_address(stored_request_body) if drop_requester_ip_address @@ -1916,7 +1988,7 @@ def _get_vector_store_request_for_spend_logs_payload( """ If user does not want to store prompts and responses, then remove the content from the vector store request metadata """ - if should_store_prompts_and_responses_in_spend_logs(): + if stores_full_prompts_in_spend_logs(): return vector_store_request_metadata # if user does not want to store prompts and responses, then remove the content from the vector store request metadata @@ -1938,7 +2010,7 @@ def _get_response_for_spend_logs_payload( payload: StandardLoggingPayload | None, kwargs: dict | None = None, ) -> str: - if payload is None: + if payload is None or stores_latest_user_message_only(): return "{}" if should_store_prompts_and_responses_in_spend_logs(): response_obj: object = payload.get("response") diff --git a/tests/integration/spend/test_spend_log_latest_user_prompt.py b/tests/integration/spend/test_spend_log_latest_user_prompt.py new file mode 100644 index 00000000000..d77dbefaabf --- /dev/null +++ b/tests/integration/spend/test_spend_log_latest_user_prompt.py @@ -0,0 +1,270 @@ +import json +from collections.abc import Callable +from pathlib import Path +from typing import Final +from uuid import uuid4 + +import pytest +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +from litellm.responses.utils import ResponsesAPIRequestUtils + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +OPENAI_MODEL: Final = "openai/gpt-4o-mini" +ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-4-5-20250929" +PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +ROW_COLUMNS: Final = "request_id, proxy_server_request, response, status, total_tokens, model" + + +def _latest_only_config(tmp_path: Path) -> Path: + config: Final = tmp_path / f"latest-user-prompt-{uuid4()}.json" + config.write_text( + json.dumps( + { + "model_list": [], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_responses_id_security": True, + "store_model_in_db": True, + "store_prompts_in_spend_logs": True, + "spend_logs_prompt_storage": "latest_user_message", + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + }, + } + ) + ) + return config + + +def _answering_model_listing(respond: Callable[[Request], Reply]) -> Callable[[Request], Reply]: + def answer(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + return respond(request) + + return answer + + +def _chat_completion(text: str) -> bytes: + return json.dumps( + { + "id": f"chatcmpl-latest-{uuid4()}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 11, "completion_tokens": 3, "total_tokens": 14}, + } + ).encode() + + +def _anthropic_message(text: str) -> bytes: + return json.dumps( + { + "id": f"msg-latest-{uuid4()}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 3}, + } + ).encode() + + +def _row(where: str, value: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows(f'SELECT {ROW_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {where}=%s', (value,)), + lambda values: len(values) == 1, + seconds=70, + ) + return rows[0] + + +def _responses_request_id(response_id: str) -> str: + decoded: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(response_id) + request_id: Final = decoded.get("response_id") + return string_value(request_id) if isinstance(request_id, str) else response_id + + +def _assert_usage_row(row: dict[str, JsonValue], expected_request: dict[str, JsonValue]) -> None: + assert object_value(row["proxy_server_request"]) == expected_request, row + assert object_value(row["response"]) == {}, row + assert row["status"] == "success", row + assert row["total_tokens"] == 14, row + + +CHAT_HISTORY: Final = [ + {"role": "system", "content": "system prompt must not be stored"}, + {"role": "user", "content": "earlier user turn must not be stored"}, + {"role": "assistant", "content": "earlier assistant turn must not be stored"}, +] + + +@pytest.mark.parametrize( + ("latest", "expected"), + [ + pytest.param( + { + "role": "user", + "content": [ + {"type": "text", "text": "latest chat question"}, + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{PIXEL}"}}, + ], + }, + {"latest_user_message": "latest chat question\n[image omitted]"}, + id="user-text-and-image", + ), + pytest.param( + {"role": "tool", "tool_call_id": "call_1", "content": "tool output must not be stored"}, + {}, + id="tool-result-turn", + ), + ], +) +def test_chat_row_keeps_only_latest_user_prompt( + gateway: Gateway, tmp_path: Path, latest: dict[str, JsonValue], expected: dict[str, JsonValue] +) -> None: + with ( + wire_server(_answering_model_listing(lambda _: Reply(body=_chat_completion("chat answer")))) as wire, + owned_proxy(gateway, tmp_path, {}, config=_latest_only_config(tmp_path)) as isolated, + isolated.scenario() as scenario, + ): + model: Final = scenario.model(model=OPENAI_MODEL, api_base=wire.url, api_key="synthetic-openai-key") + response: Final = isolated.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [*CHAT_HISTORY, latest], + "tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}], + }, + ) + assert response.status_code == 200, response.text + _assert_usage_row(_row("request_id", string_value(JSON_OBJECT.validate_json(response.content)["id"])), expected) + + +@pytest.mark.parametrize( + ("latest_content", "expected"), + [ + pytest.param( + [ + {"type": "text", "text": "latest messages question"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PIXEL}}, + ], + {"latest_user_message": "latest messages question\n[image omitted]"}, + id="user-text-and-image", + ), + pytest.param( + [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "tool output must not be stored"}], + {}, + id="tool-result-turn", + ), + ], +) +def test_messages_row_keeps_only_latest_user_prompt( + gateway: Gateway, tmp_path: Path, latest_content: list[JsonValue], expected: dict[str, JsonValue] +) -> None: + with ( + wire_server(_answering_model_listing(lambda _: Reply(body=_anthropic_message("messages answer")))) as wire, + owned_proxy(gateway, tmp_path, {}, config=_latest_only_config(tmp_path)) as isolated, + isolated.scenario() as scenario, + ): + model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key") + response: Final = isolated.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 64, + "system": "system prompt must not be stored", + "messages": [ + {"role": "user", "content": "earlier user turn must not be stored"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": {}}], + }, + {"role": "user", "content": latest_content}, + ], + }, + ) + assert response.status_code == 200, response.text + _assert_usage_row(_row("request_id", string_value(JSON_OBJECT.validate_json(response.content)["id"])), expected) + + +@pytest.mark.parametrize( + ("latest", "expected"), + [ + pytest.param( + { + "role": "user", + "content": [ + {"type": "input_text", "text": "latest responses question"}, + {"type": "input_image", "image_url": f"data:image/png;base64,{PIXEL}"}, + ], + }, + {"latest_user_message": "latest responses question\n[image omitted]"}, + id="user-text-and-image", + ), + pytest.param( + {"type": "function_call_output", "call_id": "call_1", "output": "tool output must not be stored"}, + {}, + id="tool-result-turn", + ), + ], +) +def test_responses_row_keeps_only_latest_user_prompt( + gateway: Gateway, tmp_path: Path, latest: dict[str, JsonValue], expected: dict[str, JsonValue] +) -> None: + with ( + wire_server(_answering_model_listing(lambda _: Reply(body=_anthropic_message("responses answer")))) as wire, + owned_proxy(gateway, tmp_path, {}, config=_latest_only_config(tmp_path)) as isolated, + isolated.scenario() as scenario, + ): + model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key") + response: Final = isolated.request( + "POST", + "/v1/responses", + { + "model": model, + "instructions": "system prompt must not be stored", + "input": [ + {"role": "user", "content": "earlier user turn must not be stored"}, + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + latest, + ], + }, + ) + assert response.status_code == 200, response.text + response_id: Final = string_value(object_value(response.json())["id"]) + _assert_usage_row(_row("request_id", _responses_request_id(response_id)), expected) + + +def test_failed_chat_row_keeps_only_latest_user_prompt(gateway: Gateway, tmp_path: Path) -> None: + end_user: Final = f"latest-prompt-failure-{uuid4()}" + failure: Final = Reply( + status=400, body=b'{"error":{"message":"scripted rejection","type":"invalid_request_error"}}' + ) + with ( + wire_server(_answering_model_listing(lambda _: failure)) as wire, + owned_proxy(gateway, tmp_path, {}, config=_latest_only_config(tmp_path)) as isolated, + isolated.scenario() as scenario, + ): + model: Final = scenario.model(model=OPENAI_MODEL, api_base=wire.url, api_key="synthetic-openai-key") + response: Final = isolated.request( + "POST", + "/v1/chat/completions", + {"model": model, "user": end_user, "messages": [*CHAT_HISTORY, {"role": "user", "content": "failed ask"}]}, + ) + assert response.status_code == 400, response.text + row: Final = _row("end_user", end_user) + assert row["status"] == "failure", row + assert object_value(row["proxy_server_request"]) == {"latest_user_message": "failed ask"}, row + assert object_value(row["response"]) == {}, row diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 86d064050eb..4635ab0783d 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -74,6 +74,8 @@ from litellm.types.utils import ( StandardLoggingModelInformation, StandardLoggingPayload, StandardLoggingPayloadErrorInformation, + StandardLoggingGuardrailInformation, + StandardLoggingVectorStoreRequest, ) @@ -6989,6 +6991,198 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch: pytest.MonkeyPatch assert payload_disabled["response"] == "{}" +_LATEST_ONLY_SETTINGS: Final = {"store_prompts_in_spend_logs": True, "spend_logs_prompt_storage": "latest_user_message"} +_PRIOR_CHAT_TURNS: Final = ( + {"role": "system", "content": "system prompt"}, + {"role": "user", "content": "earlier user turn"}, + {"role": "assistant", "content": "earlier answer"}, +) + + +_LATEST_ONLY_ROW: Final = TypeAdapter(dict[str, str]) +_FULL_CHAT_ROW: Final = TypeAdapter(dict[str, list[dict[str, str]]]) +_STORED_RESPONSE: Final = TypeAdapter(object) + + +def _stored_request_json(body: Mapping[str, object]) -> str: + return _get_proxy_server_request_for_spend_logs_payload( + metadata={}, litellm_params={"proxy_server_request": {"body": body}} + ) + + +def _stored_request_body(body: Mapping[str, object]) -> dict[str, str]: + return _LATEST_ONLY_ROW.validate_json(_stored_request_json(body)) + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + ({"messages": [*_PRIOR_CHAT_TURNS, {"role": "user", "content": "latest"}]}, {"latest_user_message": "latest"}), + ( + { + "system": "system prompt", + "tools": [{"name": "lookup"}], + "messages": [ + {"role": "user", "content": "earlier user turn"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + {"type": "image", "source": {"type": "base64", "data": "AAAA"}}, + {"type": "document", "source": {"type": "base64", "data": "AAAA"}}, + ], + }, + ], + }, + {"latest_user_message": "describe\n[image omitted]\n[file omitted]"}, + ), + ( + {"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:,"}}]}]}, + {"latest_user_message": "[image omitted]"}, + ), + ( + {"instructions": "system prompt", "input": "plain responses input"}, + {"latest_user_message": "plain responses input"}, + ), + ( + { + "input": [ + {"role": "user", "content": "earlier user turn"}, + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "summarize"}, + {"type": "input_file", "file_id": "file-1"}, + ], + }, + ] + }, + {"latest_user_message": "summarize\n[file omitted]"}, + ), + ( + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": ""}, + {"type": "thinking", "thinking": "hidden"}, + {"type": "text", "text": "keep"}, + ], + } + ] + }, + {"latest_user_message": "keep"}, + ), + ({"messages": [*_PRIOR_CHAT_TURNS, {"role": "tool", "tool_call_id": "c", "content": "tool out"}]}, {}), + ( + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t", "content": "tool out"}, + {"type": "text", "text": "reminder next to a tool result"}, + ], + } + ] + }, + {}, + ), + ({"input": [{"role": "user", "content": "q"}, {"type": "function_call_output", "output": "out"}]}, {}), + ({"messages": [{"role": "user", "content": "q"}, {"role": "assistant", "content": "prefill"}]}, {}), + ( + {"messages": [{"role": "user", "content": ["plain part", {"type": "text", "text": "block part"}]}]}, + {"latest_user_message": "plain part\nblock part"}, + ), + ( + {"messages": [{"role": "user", "content": [{"type": []}, {"type": {}}, {"type": "text", "text": "latest"}]}]}, + {"latest_user_message": "latest"}, + ), + ({"messages": [{"role": "user", "content": ""}]}, {}), + ({"messages": []}, {}), + ({"model": "gpt-4o-mini"}, {}), + ], +) +def test_latest_user_message_mode_stores_only_the_latest_user_text( + monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object], expected: dict[str, str] +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", _LATEST_ONLY_SETTINGS) + assert _stored_request_body(body) == expected + + +def test_latest_user_message_mode_keeps_the_existing_string_cap(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.delenv("MAX_STRING_LENGTH_PROMPT_IN_DB", raising=False) + chat_body: Final = {"messages": [{"role": "user", "content": "a" * 1500 + "b" * 1500}]} + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + capped_today: Final = _FULL_CHAT_ROW.validate_json(_stored_request_json(chat_body))["messages"][0]["content"] + monkeypatch.setattr(proxy_server, "general_settings", _LATEST_ONLY_SETTINGS) + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in capped_today, capped_today + assert _stored_request_body(chat_body) == {"latest_user_message": capped_today} + + +def test_latest_user_message_mode_drops_the_response(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + payload: Final = cast(StandardLoggingPayload, {"response": {"choices": [{"message": {"content": "answer"}}]}}) + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + assert _STORED_RESPONSE.validate_json(_get_response_for_spend_logs_payload(payload)) == payload["response"] + monkeypatch.setattr(proxy_server, "general_settings", _LATEST_ONLY_SETTINGS) + assert _get_response_for_spend_logs_payload(payload) == "{}" + + +def test_latest_user_message_mode_keeps_prompt_text_out_of_spend_log_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + prompt: Final = "earlier turn the row must not keep" + guardrail_information: Final = cast( + list[StandardLoggingGuardrailInformation], + [ + { + "guardrail_name": "pii", + "guardrail_request": {"messages": [{"role": "user", "content": prompt}]}, + "guardrail_response": prompt, + "match_details": [{"snippet": prompt}], + "classification": {"reason": prompt}, + } + ], + ) + error_information: Final = cast( + StandardLoggingPayloadErrorInformation, + {"error_message": f"BadRequestError: {{'messages': [{{'role': 'user', 'content': '{prompt}'}}]}}"}, + ) + vector_store_request: Final = cast( + list[StandardLoggingVectorStoreRequest], + [{"vector_store_search_response": {"data": [{"content": [{"type": "text", "text": prompt}]}]}}], + ) + monkeypatch.setattr(proxy_server, "general_settings", _LATEST_ONLY_SETTINGS) + + stored: Final = json.dumps( + [ + _sanitize_guardrail_information_for_spend_logs(guardrail_information), + sanitize_error_information_for_spend_logs(error_information), + _get_vector_store_request_for_spend_logs_payload(vector_store_request), + ] + ) + + assert prompt not in stored, stored + + +def test_latest_user_message_mode_unset_stores_the_full_request(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + body: Final = {"messages": [*_PRIOR_CHAT_TURNS, {"role": "user", "content": "latest"}]} + monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True}) + assert _STORED_RESPONSE.validate_json(_stored_request_json(body)) == body + + @pytest.mark.parametrize( ("configured", "disabled"), [ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f114c1c73d2..7ebdc18240e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30362,6 +30362,11 @@ export interface components { spend_capture_rate_check?: components["schemas"]["SpendCaptureRateCheckSettings"] | null; /** @description Which keys of LiteLLM_SpendLogs.metadata are written to the database. Set exactly one of 'include' (write only these keys) or 'exclude' (drop these keys). 'status' and 'cold_storage_object_key' are always written. Daily spend tables, budgets and logging callbacks still see every key. Unset writes every key */ spend_logs_metadata_fields?: components["schemas"]["SpendLogsMetadataFields"] | null; + /** + * Spend Logs Prompt Storage + * @description With store_prompts_in_spend_logs on, 'latest_user_message' stores only the text of the request's latest user message and no response. Unset stores the full request and response. + */ + spend_logs_prompt_storage?: "latest_user_message" | null; /** * Store Model In Db * @description If True, models and config are stored in and loaded from the database. Default is False.