mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 147bc104c2 into d037d86a94
This commit is contained in:
commit
0bdc63b17a
5 changed files with 551 additions and 6 deletions
|
|
@ -2833,6 +2833,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
None,
|
||||
description="If True, stores request messages and responses in spend logs. Default is False.",
|
||||
)
|
||||
spend_logs_prompt_storage: Literal["latest_user_message"] | None = Field(
|
||||
None,
|
||||
description="With store_prompts_in_spend_logs on, 'latest_user_message' stores only the text of the request's latest user message and no response. Unset stores the full request and response.",
|
||||
)
|
||||
claude_code_marketplace_auth_required: bool = Field(
|
||||
False,
|
||||
description="If True, GET /claude-code/marketplace.json requires a LiteLLM key in the Authorization or x-litellm-api-key header (?key= is not accepted) and answers 401 with WWW-Authenticate: Bearer without one, and GET /public/skill_hub returns an empty list. Default is False, which serves the enabled plugins to anyone.",
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ from litellm.litellm_core_utils.litellm_logging import (
|
|||
is_valid_sha256_hash,
|
||||
request_model_access_groups_from_litellm_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
|
||||
from litellm.litellm_core_utils.ptu_pricing import azure_spillover
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps, strip_null_bytes
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
|
||||
|
|
@ -1198,7 +1199,7 @@ def _get_messages_for_spend_logs_payload(
|
|||
metadata: dict | None = None,
|
||||
) -> str:
|
||||
if should_store_prompts_and_responses_in_spend_logs():
|
||||
if standard_logging_payload is not None:
|
||||
if standard_logging_payload is not None and not stores_latest_user_message_only():
|
||||
call_type: Final = standard_logging_payload.get("call_type", "")
|
||||
if call_type == "_arealtime":
|
||||
messages: Final = standard_logging_payload.get("messages")
|
||||
|
|
@ -1214,6 +1215,77 @@ _SENSITIVE_REQUEST_BODY_KEYS: Final = frozenset({"secret_fields"})
|
|||
_REQUEST_BODY_CREDENTIAL_MASKER: Final = SensitiveDataMasker(extra_sensitive_patterns=frozenset({"apikey"}))
|
||||
_TOOL_INPUT_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use", "mcp_tool_use"})
|
||||
_TOOL_OUTPUT_BLOCK_TYPES: Final = frozenset({"tool_result", "mcp_tool_result", "function_call_output"})
|
||||
_LATEST_USER_MESSAGE_STORAGE: Final = "latest_user_message"
|
||||
_TEXT_BLOCK_TYPES: Final = frozenset({"text", "input_text"})
|
||||
_OMITTED_BLOCK_PLACEHOLDERS: Final = MappingProxyType(
|
||||
{
|
||||
"image": "[image omitted]",
|
||||
"image_url": "[image omitted]",
|
||||
"input_image": "[image omitted]",
|
||||
"file": "[file omitted]",
|
||||
"input_file": "[file omitted]",
|
||||
"document": "[file omitted]",
|
||||
"input_audio": "[audio omitted]",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def stores_latest_user_message_only() -> bool:
|
||||
from litellm.proxy.proxy_server import general_settings_view
|
||||
|
||||
return general_settings_view().get("spend_logs_prompt_storage") == _LATEST_USER_MESSAGE_STORAGE
|
||||
|
||||
|
||||
def stores_full_prompts_in_spend_logs() -> bool:
|
||||
return should_store_prompts_and_responses_in_spend_logs() and not stores_latest_user_message_only()
|
||||
|
||||
|
||||
def _block_type(block: object) -> str | None:
|
||||
block_type: Final = message_field(block, "type")
|
||||
return block_type if isinstance(block_type, str) else None
|
||||
|
||||
|
||||
def _content_block_text(block: object) -> str | None:
|
||||
if isinstance(block, str):
|
||||
return block
|
||||
block_type: Final = _block_type(block)
|
||||
if block_type in _TEXT_BLOCK_TYPES:
|
||||
text: Final = message_field(block, "text")
|
||||
return text if isinstance(text, str) else None
|
||||
return _OMITTED_BLOCK_PLACEHOLDERS.get(block_type) if block_type is not None else None
|
||||
|
||||
|
||||
def _is_tool_output_block(block: object) -> bool:
|
||||
return _block_type(block) in _TOOL_OUTPUT_BLOCK_TYPES
|
||||
|
||||
|
||||
def _user_content_text(content: object) -> str | None:
|
||||
if isinstance(content, str):
|
||||
return content or None
|
||||
blocks: Final = parts_of(content)
|
||||
if any(_is_tool_output_block(block) for block in blocks):
|
||||
return None
|
||||
texts: Final = tuple(text for text in map(_content_block_text, blocks) if text)
|
||||
return "\n".join(texts) or None
|
||||
|
||||
|
||||
def _latest_user_message_text(request_body: Mapping[str, object]) -> str | None:
|
||||
turns: Final = request_body.get("messages", request_body.get("input"))
|
||||
if isinstance(turns, str):
|
||||
return turns or None
|
||||
latest: Final = next(reversed(parts_of(turns)), None)
|
||||
if message_field(latest, "role") != "user":
|
||||
return None
|
||||
return _user_content_text(message_field(latest, "content"))
|
||||
|
||||
|
||||
def _latest_user_message_body(request_body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
text: Final = _latest_user_message_text(request_body)
|
||||
return MappingProxyType({}) if text is None else MappingProxyType({_LATEST_USER_MESSAGE_STORAGE: text})
|
||||
|
||||
|
||||
def _stored_request_body(request_body: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return _latest_user_message_body(request_body) if stores_latest_user_message_only() else request_body
|
||||
|
||||
|
||||
def _is_request_body_credential(key: str, value: object) -> bool:
|
||||
|
|
@ -1476,7 +1548,7 @@ def _sanitize_guardrail_information_for_spend_logs(
|
|||
here to match OTEL's defensive read pattern; otherwise iteration would
|
||||
yield the dict's keys and crash the whole spend-log write.
|
||||
"""
|
||||
if guardrail_information is None or should_store_prompts_and_responses_in_spend_logs():
|
||||
if guardrail_information is None or stores_full_prompts_in_spend_logs():
|
||||
return guardrail_information
|
||||
entries: Final = [guardrail_information] if isinstance(guardrail_information, dict) else guardrail_information
|
||||
return [_redact_prompt_fields_in_guardrail_entry(entry) for entry in entries if isinstance(entry, dict)]
|
||||
|
|
@ -1554,7 +1626,7 @@ def sanitize_error_information_for_spend_logs(
|
|||
)
|
||||
sanitized = cast(dict, {**persisted})
|
||||
|
||||
if not should_store_prompts_and_responses_in_spend_logs():
|
||||
if not stores_full_prompts_in_spend_logs():
|
||||
for field in ("error_message", "traceback"):
|
||||
value = sanitized.get(field)
|
||||
if isinstance(value, str):
|
||||
|
|
@ -1895,7 +1967,7 @@ def _get_proxy_server_request_for_spend_logs_payload(
|
|||
_request_body = _convert_mapping_to_json_serializable(without_classifier_audit(_request_body))
|
||||
perform_redaction(model_call_details=_request_body, result=None)
|
||||
|
||||
stored_request_body: Final = _sanitize_proxy_server_request_body(_request_body)
|
||||
stored_request_body: Final = _sanitize_proxy_server_request_body(_stored_request_body(_request_body))
|
||||
_request_body_json_str: Final = safe_dumps(
|
||||
_stored_request_without_requester_ip_address(stored_request_body)
|
||||
if drop_requester_ip_address
|
||||
|
|
@ -1916,7 +1988,7 @@ def _get_vector_store_request_for_spend_logs_payload(
|
|||
"""
|
||||
If user does not want to store prompts and responses, then remove the content from the vector store request metadata
|
||||
"""
|
||||
if should_store_prompts_and_responses_in_spend_logs():
|
||||
if stores_full_prompts_in_spend_logs():
|
||||
return vector_store_request_metadata
|
||||
|
||||
# if user does not want to store prompts and responses, then remove the content from the vector store request metadata
|
||||
|
|
@ -1938,7 +2010,7 @@ def _get_response_for_spend_logs_payload(
|
|||
payload: StandardLoggingPayload | None,
|
||||
kwargs: dict | None = None,
|
||||
) -> str:
|
||||
if payload is None:
|
||||
if payload is None or stores_latest_user_message_only():
|
||||
return "{}"
|
||||
if should_store_prompts_and_responses_in_spend_logs():
|
||||
response_obj: object = payload.get("response")
|
||||
|
|
|
|||
270
tests/integration/spend/test_spend_log_latest_user_prompt.py
Normal file
270
tests/integration/spend/test_spend_log_latest_user_prompt.py
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
import json
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
OPENAI_MODEL: Final = "openai/gpt-4o-mini"
|
||||
ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-4-5-20250929"
|
||||
PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||
ROW_COLUMNS: Final = "request_id, proxy_server_request, response, status, total_tokens, model"
|
||||
|
||||
|
||||
def _latest_only_config(tmp_path: Path) -> Path:
|
||||
config: Final = tmp_path / f"latest-user-prompt-{uuid4()}.json"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [],
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
"disable_responses_id_security": True,
|
||||
"store_model_in_db": True,
|
||||
"store_prompts_in_spend_logs": True,
|
||||
"spend_logs_prompt_storage": "latest_user_message",
|
||||
"proxy_batch_write_at": 1,
|
||||
"proxy_batch_polling_interval": 1,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _answering_model_listing(respond: Callable[[Request], Reply]) -> Callable[[Request], Reply]:
|
||||
def answer(request: Request) -> Reply:
|
||||
if request.method == "GET":
|
||||
return Reply(body=b'{"object":"list","data":[]}')
|
||||
return respond(request)
|
||||
|
||||
return answer
|
||||
|
||||
|
||||
def _chat_completion(text: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": f"chatcmpl-latest-{uuid4()}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 11, "completion_tokens": 3, "total_tokens": 14},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _anthropic_message(text: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": f"msg-latest-{uuid4()}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5-20250929",
|
||||
"content": [{"type": "text", "text": text}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 11, "output_tokens": 3},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _row(where: str, value: str) -> dict[str, JsonValue]:
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(f'SELECT {ROW_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {where}=%s', (value,)),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _responses_request_id(response_id: str) -> str:
|
||||
decoded: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(response_id)
|
||||
request_id: Final = decoded.get("response_id")
|
||||
return string_value(request_id) if isinstance(request_id, str) else response_id
|
||||
|
||||
|
||||
def _assert_usage_row(row: dict[str, JsonValue], expected_request: dict[str, JsonValue]) -> None:
|
||||
assert object_value(row["proxy_server_request"]) == expected_request, row
|
||||
assert object_value(row["response"]) == {}, row
|
||||
assert row["status"] == "success", row
|
||||
assert row["total_tokens"] == 14, row
|
||||
|
||||
|
||||
CHAT_HISTORY: Final = [
|
||||
{"role": "system", "content": "system prompt must not be stored"},
|
||||
{"role": "user", "content": "earlier user turn must not be stored"},
|
||||
{"role": "assistant", "content": "earlier assistant turn must not be stored"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("latest", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "latest chat question"},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{PIXEL}"}},
|
||||
],
|
||||
},
|
||||
{"latest_user_message": "latest chat question\n[image omitted]"},
|
||||
id="user-text-and-image",
|
||||
),
|
||||
pytest.param(
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "tool output must not be stored"},
|
||||
{},
|
||||
id="tool-result-turn",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_chat_row_keeps_only_latest_user_prompt(
|
||||
gateway: Gateway, tmp_path: Path, latest: dict[str, JsonValue], expected: dict[str, JsonValue]
|
||||
) -> None:
|
||||
with (
|
||||
wire_server(_answering_model_listing(lambda _: Reply(body=_chat_completion("chat answer")))) as wire,
|
||||
owned_proxy(gateway, tmp_path, {}, config=_latest_only_config(tmp_path)) as isolated,
|
||||
isolated.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(model=OPENAI_MODEL, api_base=wire.url, api_key="synthetic-openai-key")
|
||||
response: Final = isolated.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [*CHAT_HISTORY, latest],
|
||||
"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
_assert_usage_row(_row("request_id", string_value(JSON_OBJECT.validate_json(response.content)["id"])), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("latest_content", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
[
|
||||
{"type": "text", "text": "latest messages question"},
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PIXEL}},
|
||||
],
|
||||
{"latest_user_message": "latest messages question\n[image omitted]"},
|
||||
id="user-text-and-image",
|
||||
),
|
||||
pytest.param(
|
||||
[{"type": "tool_result", "tool_use_id": "toolu_1", "content": "tool output must not be stored"}],
|
||||
{},
|
||||
id="tool-result-turn",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_messages_row_keeps_only_latest_user_prompt(
|
||||
gateway: Gateway, tmp_path: Path, latest_content: list[JsonValue], expected: dict[str, JsonValue]
|
||||
) -> None:
|
||||
with (
|
||||
wire_server(_answering_model_listing(lambda _: Reply(body=_anthropic_message("messages answer")))) as wire,
|
||||
owned_proxy(gateway, tmp_path, {}, config=_latest_only_config(tmp_path)) as isolated,
|
||||
isolated.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
|
||||
response: Final = isolated.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"system": "system prompt must not be stored",
|
||||
"messages": [
|
||||
{"role": "user", "content": "earlier user turn must not be stored"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [{"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": {}}],
|
||||
},
|
||||
{"role": "user", "content": latest_content},
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
_assert_usage_row(_row("request_id", string_value(JSON_OBJECT.validate_json(response.content)["id"])), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("latest", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "latest responses question"},
|
||||
{"type": "input_image", "image_url": f"data:image/png;base64,{PIXEL}"},
|
||||
],
|
||||
},
|
||||
{"latest_user_message": "latest responses question\n[image omitted]"},
|
||||
id="user-text-and-image",
|
||||
),
|
||||
pytest.param(
|
||||
{"type": "function_call_output", "call_id": "call_1", "output": "tool output must not be stored"},
|
||||
{},
|
||||
id="tool-result-turn",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_responses_row_keeps_only_latest_user_prompt(
|
||||
gateway: Gateway, tmp_path: Path, latest: dict[str, JsonValue], expected: dict[str, JsonValue]
|
||||
) -> None:
|
||||
with (
|
||||
wire_server(_answering_model_listing(lambda _: Reply(body=_anthropic_message("responses answer")))) as wire,
|
||||
owned_proxy(gateway, tmp_path, {}, config=_latest_only_config(tmp_path)) as isolated,
|
||||
isolated.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(model=ANTHROPIC_MODEL, api_base=wire.url, api_key="synthetic-anthropic-key")
|
||||
response: Final = isolated.request(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{
|
||||
"model": model,
|
||||
"instructions": "system prompt must not be stored",
|
||||
"input": [
|
||||
{"role": "user", "content": "earlier user turn must not be stored"},
|
||||
{"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"},
|
||||
latest,
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
response_id: Final = string_value(object_value(response.json())["id"])
|
||||
_assert_usage_row(_row("request_id", _responses_request_id(response_id)), expected)
|
||||
|
||||
|
||||
def test_failed_chat_row_keeps_only_latest_user_prompt(gateway: Gateway, tmp_path: Path) -> None:
|
||||
end_user: Final = f"latest-prompt-failure-{uuid4()}"
|
||||
failure: Final = Reply(
|
||||
status=400, body=b'{"error":{"message":"scripted rejection","type":"invalid_request_error"}}'
|
||||
)
|
||||
with (
|
||||
wire_server(_answering_model_listing(lambda _: failure)) as wire,
|
||||
owned_proxy(gateway, tmp_path, {}, config=_latest_only_config(tmp_path)) as isolated,
|
||||
isolated.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(model=OPENAI_MODEL, api_base=wire.url, api_key="synthetic-openai-key")
|
||||
response: Final = isolated.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "user": end_user, "messages": [*CHAT_HISTORY, {"role": "user", "content": "failed ask"}]},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
row: Final = _row("end_user", end_user)
|
||||
assert row["status"] == "failure", row
|
||||
assert object_value(row["proxy_server_request"]) == {"latest_user_message": "failed ask"}, row
|
||||
assert object_value(row["response"]) == {}, row
|
||||
|
|
@ -74,6 +74,8 @@ from litellm.types.utils import (
|
|||
StandardLoggingModelInformation,
|
||||
StandardLoggingPayload,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
StandardLoggingGuardrailInformation,
|
||||
StandardLoggingVectorStoreRequest,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -6989,6 +6991,198 @@ def test_spend_logs_payload_with_prompts_enabled(monkeypatch: pytest.MonkeyPatch
|
|||
assert payload_disabled["response"] == "{}"
|
||||
|
||||
|
||||
_LATEST_ONLY_SETTINGS: Final = {"store_prompts_in_spend_logs": True, "spend_logs_prompt_storage": "latest_user_message"}
|
||||
_PRIOR_CHAT_TURNS: Final = (
|
||||
{"role": "system", "content": "system prompt"},
|
||||
{"role": "user", "content": "earlier user turn"},
|
||||
{"role": "assistant", "content": "earlier answer"},
|
||||
)
|
||||
|
||||
|
||||
_LATEST_ONLY_ROW: Final = TypeAdapter(dict[str, str])
|
||||
_FULL_CHAT_ROW: Final = TypeAdapter(dict[str, list[dict[str, str]]])
|
||||
_STORED_RESPONSE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _stored_request_json(body: Mapping[str, object]) -> str:
|
||||
return _get_proxy_server_request_for_spend_logs_payload(
|
||||
metadata={}, litellm_params={"proxy_server_request": {"body": body}}
|
||||
)
|
||||
|
||||
|
||||
def _stored_request_body(body: Mapping[str, object]) -> dict[str, str]:
|
||||
return _LATEST_ONLY_ROW.validate_json(_stored_request_json(body))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "expected"),
|
||||
[
|
||||
({"messages": [*_PRIOR_CHAT_TURNS, {"role": "user", "content": "latest"}]}, {"latest_user_message": "latest"}),
|
||||
(
|
||||
{
|
||||
"system": "system prompt",
|
||||
"tools": [{"name": "lookup"}],
|
||||
"messages": [
|
||||
{"role": "user", "content": "earlier user turn"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "describe"},
|
||||
{"type": "image", "source": {"type": "base64", "data": "AAAA"}},
|
||||
{"type": "document", "source": {"type": "base64", "data": "AAAA"}},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{"latest_user_message": "describe\n[image omitted]\n[file omitted]"},
|
||||
),
|
||||
(
|
||||
{"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:,"}}]}]},
|
||||
{"latest_user_message": "[image omitted]"},
|
||||
),
|
||||
(
|
||||
{"instructions": "system prompt", "input": "plain responses input"},
|
||||
{"latest_user_message": "plain responses input"},
|
||||
),
|
||||
(
|
||||
{
|
||||
"input": [
|
||||
{"role": "user", "content": "earlier user turn"},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "summarize"},
|
||||
{"type": "input_file", "file_id": "file-1"},
|
||||
],
|
||||
},
|
||||
]
|
||||
},
|
||||
{"latest_user_message": "summarize\n[file omitted]"},
|
||||
),
|
||||
(
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": ""},
|
||||
{"type": "thinking", "thinking": "hidden"},
|
||||
{"type": "text", "text": "keep"},
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
{"latest_user_message": "keep"},
|
||||
),
|
||||
({"messages": [*_PRIOR_CHAT_TURNS, {"role": "tool", "tool_call_id": "c", "content": "tool out"}]}, {}),
|
||||
(
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "tool_result", "tool_use_id": "t", "content": "tool out"},
|
||||
{"type": "text", "text": "reminder next to a tool result"},
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
{},
|
||||
),
|
||||
({"input": [{"role": "user", "content": "q"}, {"type": "function_call_output", "output": "out"}]}, {}),
|
||||
({"messages": [{"role": "user", "content": "q"}, {"role": "assistant", "content": "prefill"}]}, {}),
|
||||
(
|
||||
{"messages": [{"role": "user", "content": ["plain part", {"type": "text", "text": "block part"}]}]},
|
||||
{"latest_user_message": "plain part\nblock part"},
|
||||
),
|
||||
(
|
||||
{"messages": [{"role": "user", "content": [{"type": []}, {"type": {}}, {"type": "text", "text": "latest"}]}]},
|
||||
{"latest_user_message": "latest"},
|
||||
),
|
||||
({"messages": [{"role": "user", "content": ""}]}, {}),
|
||||
({"messages": []}, {}),
|
||||
({"model": "gpt-4o-mini"}, {}),
|
||||
],
|
||||
)
|
||||
def test_latest_user_message_mode_stores_only_the_latest_user_text(
|
||||
monkeypatch: pytest.MonkeyPatch, body: Mapping[str, object], expected: dict[str, str]
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.setattr(proxy_server, "general_settings", _LATEST_ONLY_SETTINGS)
|
||||
assert _stored_request_body(body) == expected
|
||||
|
||||
|
||||
def test_latest_user_message_mode_keeps_the_existing_string_cap(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
monkeypatch.delenv("MAX_STRING_LENGTH_PROMPT_IN_DB", raising=False)
|
||||
chat_body: Final = {"messages": [{"role": "user", "content": "a" * 1500 + "b" * 1500}]}
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
|
||||
capped_today: Final = _FULL_CHAT_ROW.validate_json(_stored_request_json(chat_body))["messages"][0]["content"]
|
||||
monkeypatch.setattr(proxy_server, "general_settings", _LATEST_ONLY_SETTINGS)
|
||||
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in capped_today, capped_today
|
||||
assert _stored_request_body(chat_body) == {"latest_user_message": capped_today}
|
||||
|
||||
|
||||
def test_latest_user_message_mode_drops_the_response(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
payload: Final = cast(StandardLoggingPayload, {"response": {"choices": [{"message": {"content": "answer"}}]}})
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
|
||||
assert _STORED_RESPONSE.validate_json(_get_response_for_spend_logs_payload(payload)) == payload["response"]
|
||||
monkeypatch.setattr(proxy_server, "general_settings", _LATEST_ONLY_SETTINGS)
|
||||
assert _get_response_for_spend_logs_payload(payload) == "{}"
|
||||
|
||||
|
||||
def test_latest_user_message_mode_keeps_prompt_text_out_of_spend_log_metadata(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
prompt: Final = "earlier turn the row must not keep"
|
||||
guardrail_information: Final = cast(
|
||||
list[StandardLoggingGuardrailInformation],
|
||||
[
|
||||
{
|
||||
"guardrail_name": "pii",
|
||||
"guardrail_request": {"messages": [{"role": "user", "content": prompt}]},
|
||||
"guardrail_response": prompt,
|
||||
"match_details": [{"snippet": prompt}],
|
||||
"classification": {"reason": prompt},
|
||||
}
|
||||
],
|
||||
)
|
||||
error_information: Final = cast(
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
{"error_message": f"BadRequestError: {{'messages': [{{'role': 'user', 'content': '{prompt}'}}]}}"},
|
||||
)
|
||||
vector_store_request: Final = cast(
|
||||
list[StandardLoggingVectorStoreRequest],
|
||||
[{"vector_store_search_response": {"data": [{"content": [{"type": "text", "text": prompt}]}]}}],
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", _LATEST_ONLY_SETTINGS)
|
||||
|
||||
stored: Final = json.dumps(
|
||||
[
|
||||
_sanitize_guardrail_information_for_spend_logs(guardrail_information),
|
||||
sanitize_error_information_for_spend_logs(error_information),
|
||||
_get_vector_store_request_for_spend_logs_payload(vector_store_request),
|
||||
]
|
||||
)
|
||||
|
||||
assert prompt not in stored, stored
|
||||
|
||||
|
||||
def test_latest_user_message_mode_unset_stores_the_full_request(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
body: Final = {"messages": [*_PRIOR_CHAT_TURNS, {"role": "user", "content": "latest"}]}
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {"store_prompts_in_spend_logs": True})
|
||||
assert _STORED_RESPONSE.validate_json(_stored_request_json(body)) == body
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured", "disabled"),
|
||||
[
|
||||
|
|
|
|||
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
5
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -30362,6 +30362,11 @@ export interface components {
|
|||
spend_capture_rate_check?: components["schemas"]["SpendCaptureRateCheckSettings"] | null;
|
||||
/** @description Which keys of LiteLLM_SpendLogs.metadata are written to the database. Set exactly one of 'include' (write only these keys) or 'exclude' (drop these keys). 'status' and 'cold_storage_object_key' are always written. Daily spend tables, budgets and logging callbacks still see every key. Unset writes every key */
|
||||
spend_logs_metadata_fields?: components["schemas"]["SpendLogsMetadataFields"] | null;
|
||||
/**
|
||||
* Spend Logs Prompt Storage
|
||||
* @description With store_prompts_in_spend_logs on, 'latest_user_message' stores only the text of the request's latest user message and no response. Unset stores the full request and response.
|
||||
*/
|
||||
spend_logs_prompt_storage?: "latest_user_message" | null;
|
||||
/**
|
||||
* Store Model In Db
|
||||
* @description If True, models and config are stored in and loaded from the database. Default is False.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue