scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge 3eebf2bd3e into c2c2a623c0

Browse source
This commit is contained in:
spawrks 2026-09-13 15:32:12 +00:00 • committed by GitHub
commit 207018c427
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 268 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
litellm/proxy/_types.py
View file

@ -2754,6 +2754,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
None,
description="If True, stores request messages and responses in spend logs. Default is False.",
)
store_responses_in_spend_logs: bool | None = Field(
None,
description="Controls response content storage in spend logs independently. When unset, follows store_prompts_in_spend_logs for backward compatibility.",
)
disable_auto_add_proxy_admin_to_teams: bool | None = Field(
None,
description="By default, the user calling /team/new is automatically added to the new team as a team admin. If True, proxy admins are no longer auto-added; members explicitly listed in members_with_roles are unaffected. Default is False.",

16
litellm/proxy/proxy_server.py
View file

@ -7107,6 +7107,21 @@ class ProxyConfig:
# For other types, convert to bool
general_settings["store_prompts_in_spend_logs"] = bool(value)
if "store_responses_in_spend_logs" in _general_settings:
response_storage_value: Final = (
general_settings.get("store_responses_in_spend_logs")
if "store_responses_in_spend_logs" in self._yaml_general_settings_keys
else _general_settings["store_responses_in_spend_logs"]
)
if response_storage_value is None:
general_settings["store_responses_in_spend_logs"] = None
elif isinstance(response_storage_value, bool):
general_settings["store_responses_in_spend_logs"] = response_storage_value
elif isinstance(response_storage_value, str):
general_settings["store_responses_in_spend_logs"] = response_storage_value.lower() == "true"
else:
general_settings["store_responses_in_spend_logs"] = bool(response_storage_value)
if "disable_auto_add_proxy_admin_to_teams" in _general_settings:
value = _general_settings["disable_auto_add_proxy_admin_to_teams"]
if isinstance(value, str):
@ -17042,6 +17057,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro
"pass_through_endpoints": "PydanticModel",
"store_model_in_db": "Boolean",
"store_prompts_in_spend_logs": "Boolean",
"store_responses_in_spend_logs": "Boolean",
"maximum_spend_logs_retention_period": "String",
"maximum_health_check_retention_period": "String",
"maximum_spend_logs_cleanup_batch_size": "Integer",

60
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -53,7 +53,6 @@ from litellm.types.utils import (
StandardLoggingPayload,
StandardLoggingPayloadErrorInformation,
StandardLoggingVectorStoreRequest,
VectorStoreSearchResponse,
)
from litellm.utils import get_end_user_id_for_cost_tracking
@ -199,14 +198,19 @@ def _get_spend_logs_metadata(
_already_redacted: Final = (
isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == _raw_key
)
store_responses: Final = _should_store_responses_in_spend_logs()
clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_redacted=_already_redacted)
clean_metadata["applied_guardrails"] = applied_guardrails
clean_metadata["batch_models"] = batch_models
clean_metadata["batch_successful_requests"] = batch_successful_requests
clean_metadata["batch_failed_requests"] = batch_failed_requests
clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata
clean_metadata["mcp_tool_call_metadata"] = _get_mcp_tool_call_metadata_for_spend_logs_payload(
mcp_tool_call_metadata,
store_responses=store_responses,
)
clean_metadata["vector_store_request_metadata"] = _get_vector_store_request_for_spend_logs_payload(
vector_store_request_metadata
vector_store_request_metadata,
store_responses=store_responses,
)
clean_metadata["guardrail_information"] = _sanitize_guardrail_information_for_spend_logs(guardrail_information)
clean_metadata["usage_object"] = usage_object
@ -1322,28 +1326,36 @@ def _get_proxy_server_request_for_spend_logs_payload(
return "{}"
def _get_mcp_tool_call_metadata_for_spend_logs_payload(
mcp_tool_call_metadata: StandardLoggingMCPToolCall | None,
*,
store_responses: bool,
) -> StandardLoggingMCPToolCall | None:
if mcp_tool_call_metadata is None or store_responses:
return mcp_tool_call_metadata
return cast(
StandardLoggingMCPToolCall,
{key: value for key, value in mcp_tool_call_metadata.items() if key != "result"},
)
def _get_vector_store_request_for_spend_logs_payload(
vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None,
*,
store_responses: bool,
) -> list[StandardLoggingVectorStoreRequest] | None:
"""
If user does not want to store prompts and responses, then remove the content from the vector store request metadata
"""
if should_store_prompts_and_responses_in_spend_logs():
if store_responses:
return vector_store_request_metadata
# if user does not want to store prompts and responses, then remove the content from the vector store request metadata
if vector_store_request_metadata is None:
return None
for vector_store_request in vector_store_request_metadata:
vector_store_search_response: VectorStoreSearchResponse = (
vector_store_request.get("vector_store_search_response") or VectorStoreSearchResponse()
return [
cast(
StandardLoggingVectorStoreRequest,
{key: value for key, value in vector_store_request.items() if key != "vector_store_search_response"},
)
response_data = vector_store_search_response.get("data", []) or []
for response_item in response_data:
for content_item in response_item.get("content", []) or []:
if "text" in content_item:
content_item["text"] = REDACTED_BY_LITELM_STRING
return vector_store_request_metadata
for vector_store_request in vector_store_request_metadata
]
def _get_response_for_spend_logs_payload(
@ -1352,7 +1364,7 @@ def _get_response_for_spend_logs_payload(
) -> str:
if payload is None:
return "{}"
if should_store_prompts_and_responses_in_spend_logs():
if _should_store_responses_in_spend_logs():
response_obj: object = payload.get("response")
if response_obj is None:
return "{}"
@ -1419,6 +1431,18 @@ def should_store_prompts_and_responses_in_spend_logs() -> bool:
return get_secret_bool("STORE_PROMPTS_IN_SPEND_LOGS") is True
def _should_store_responses_in_spend_logs() -> bool:
from litellm.proxy.proxy_server import general_settings
store_responses_value: Final = general_settings.get("store_responses_in_spend_logs")
if store_responses_value is not None:
if isinstance(store_responses_value, str):
return store_responses_value.lower() == "true"
return store_responses_value is True
return should_store_prompts_and_responses_in_spend_logs()
def _get_status_for_spend_log(
metadata: dict,
) -> Literal["success", "failure"]:

41
tests/proxy_unit_tests/test_proxy_config_unit_test.py
View file

@ -353,3 +353,44 @@ class TestYamlStorePromptsDbOverride:
"""_yaml_general_settings_keys should be empty on init."""
proxy_config = ProxyConfig()
assert proxy_config._yaml_general_settings_keys == set()
@pytest.mark.asyncio
async def test_response_storage_yaml_value_takes_precedence_over_db(self):
proxy_config = self._make_proxy_config_with_yaml_keys({"store_responses_in_spend_logs"})
test_general_settings = {"store_responses_in_spend_logs": False}
with mock.patch( # test-quality-ok: verifies synchronization with the process-wide settings store
"litellm.proxy.proxy_server.general_settings", test_general_settings
):
await proxy_config._update_general_settings(
db_general_settings={"store_responses_in_spend_logs": True},
)
assert test_general_settings["store_responses_in_spend_logs"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize(
("db_value", "expected"),
[
(None, None),
(True, True),
("TRUE", True),
(0, False),
],
)
async def test_response_storage_db_value_is_normalized_when_yaml_omits_key(
self,
db_value,
expected,
):
proxy_config = self._make_proxy_config_with_yaml_keys({"master_key"})
test_general_settings = {"master_key": "sk-test"}
with mock.patch( # test-quality-ok: verifies synchronization with the process-wide settings store
"litellm.proxy.proxy_server.general_settings", test_general_settings
):
await proxy_config._update_general_settings(
db_general_settings={"store_responses_in_spend_logs": db_value},
)
assert test_general_settings["store_responses_in_spend_logs"] is expected

210
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -27,6 +27,7 @@ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_get_messages_for_spend_logs_payload,
_get_mcp_tool_call_metadata_for_spend_logs_payload,
_get_proxy_server_request_for_spend_logs_payload,
_get_request_duration_ms,
_get_response_for_spend_logs_payload,
@ -39,6 +40,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
_sanitize_error_information_for_spend_logs,
_sanitize_guardrail_information_for_spend_logs,
_sanitize_request_body_for_spend_logs_payload,
_should_store_responses_in_spend_logs,
get_logging_payload,
get_spend_logs_id,
should_store_prompts_and_responses_in_spend_logs,
@ -46,9 +48,11 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
from litellm.proxy.utils import hash_token
from litellm.types.utils import (
StandardLoggingHiddenParams,
StandardLoggingMCPToolCall,
StandardLoggingMetadata,
StandardLoggingModelInformation,
StandardLoggingPayload,
StandardLoggingVectorStoreRequest,
)
@ -79,10 +83,16 @@ def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch
"classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]},
"originating_request_masked": {"input": "source-only", "api_key": "REDACTED"},
}
stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload(
metadata={}, litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}},
kwargs={"standard_logging_object": audit, "standard_callback_dynamic_params": {"turn_off_message_logging": redact}},
))
stored: Final = json.loads(
_get_proxy_server_request_for_spend_logs_payload(
metadata={},
litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}},
kwargs={
"standard_logging_object": audit,
"standard_callback_dynamic_params": {"turn_off_message_logging": redact},
},
)
)
if not store_prompts or redact:
assert "classifier_input" not in stored
assert "originating_request_masked" not in stored
@ -208,9 +218,7 @@ def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id():
from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id
create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1")
cost_session: Final = _get_batch_trace_session_id(
call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost"
)
cost_session: Final = _get_batch_trace_session_id(call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost")
assert create_session == cost_session == "batch-uid-1"
@ -656,52 +664,86 @@ def test_sanitize_request_body_for_spend_logs_payload_circular_reference():
assert sanitized == {"b": {"a": {}}} # Should return empty dict for circular reference
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true(
mock_should_store,
@pytest.mark.parametrize(
("store_responses", "expected_response_stored"),
[
(False, False),
(True, True),
],
)
def test_get_vector_store_request_for_spend_logs_payload_uses_response_setting(
store_responses: bool, expected_response_stored: bool
):
# When should_store_prompts_and_responses_in_spend_logs returns True
mock_should_store.return_value = True
# Sample vector store request metadata
vector_store_request = [
{"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}}
vector_store_request: Final[list[StandardLoggingVectorStoreRequest]] = [
{
"vector_store_id": "vs-123",
"custom_llm_provider": "openai",
"query": "request content",
"start_time": 1.0,
"end_time": 2.0,
"vector_store_search_response": {
"search_query": "request content",
"data": [
{
"file_id": "file-123",
"filename": "sensitive-filename.txt",
"attributes": {"source_url": "https://sensitive.example"},
"content": [{"text": "sensitive information", "type": "text"}],
}
],
},
}
]
# When store_prompts is True, the original data should be returned unchanged
result = _get_vector_store_request_for_spend_logs_payload(vector_store_request)
assert result == vector_store_request
assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == "sensitive information"
result: Final = _get_vector_store_request_for_spend_logs_payload(
vector_store_request,
store_responses=store_responses,
)
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false(
mock_should_store,
):
# When should_store_prompts_and_responses_in_spend_logs returns False
mock_should_store.return_value = False
# Sample vector store request metadata
vector_store_request = [
{"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}}
]
# When store_prompts is False, text should be redacted
result = _get_vector_store_request_for_spend_logs_payload(vector_store_request)
assert result is not None
assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == REDACTED_BY_LITELM_STRING
# Ensure other fields are unchanged
assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"] == "text"
assert result[0]["vector_store_id"] == "vs-123"
assert result[0]["custom_llm_provider"] == "openai"
assert result[0]["query"] == "request content"
assert result[0]["start_time"] == 1.0
assert result[0]["end_time"] == 2.0
expected_response: Final = (
vector_store_request[0]["vector_store_search_response"] if expected_response_stored else None
)
assert result[0].get("vector_store_search_response") == expected_response
assert ("vector_store_search_response" in result[0]) is expected_response_stored
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_store):
# When input is None
mock_should_store.return_value = False
result = _get_vector_store_request_for_spend_logs_payload(None)
def test_get_vector_store_request_for_spend_logs_payload_null_input():
result = _get_vector_store_request_for_spend_logs_payload(None, store_responses=False)
assert result is None
@pytest.mark.parametrize(
("store_responses", "expected_result"),
[
(False, None),
(True, {"content": "sensitive response"}),
],
)
def test_get_spend_logs_metadata_uses_response_setting_for_mcp_result(
store_responses: bool, expected_result: Mapping[str, str] | None
):
mcp_metadata: Final[StandardLoggingMCPToolCall] = {
"name": "search",
"arguments": {"query": "request content"},
"result": {"content": "sensitive response"},
}
stored_mcp_metadata: Final = _get_mcp_tool_call_metadata_for_spend_logs_payload(
mcp_metadata,
store_responses=store_responses,
)
assert stored_mcp_metadata is not None
assert stored_mcp_metadata["name"] == "search"
assert stored_mcp_metadata["arguments"] == {"query": "request content"}
assert stored_mcp_metadata.get("result") == expected_result
@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs")
def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store):
"""
@ -1708,6 +1750,78 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin
assert result is False, "Expected False (from env var) when key missing, got True"
@pytest.mark.parametrize(
("settings", "expected"),
[
({"store_prompts_in_spend_logs": True}, True),
({"store_prompts_in_spend_logs": False}, False),
(
{
"store_prompts_in_spend_logs": True,
"store_responses_in_spend_logs": False,
},
False,
),
(
{
"store_prompts_in_spend_logs": False,
"store_responses_in_spend_logs": True,
},
True,
),
(
{
"store_prompts_in_spend_logs": True,
"store_responses_in_spend_logs": "FALSE",
},
False,
),
],
)
def test_should_store_responses_in_spend_logs(
settings,
expected,
):
with patch( # test-quality-ok: isolates the process-wide settings source for retention resolution
"litellm.proxy.proxy_server.general_settings", settings
):
assert _should_store_responses_in_spend_logs() is expected
def test_spend_logs_can_store_request_without_response():
settings = {
"store_prompts_in_spend_logs": True,
"store_responses_in_spend_logs": False,
}
kwargs = {
"litellm_params": {
"proxy_server_request": {
"body": {
"model": "gpt-5-mini",
"messages": [{"role": "user", "content": "Hello!"}],
}
}
}
}
payload = cast(
StandardLoggingPayload,
{"response": {"role": "assistant", "content": "Hi there!"}},
)
with patch( # test-quality-ok: isolates the process-wide settings source for the retention integration check
"litellm.proxy.proxy_server.general_settings", settings
):
request_result = _get_proxy_server_request_for_spend_logs_payload(
metadata={},
litellm_params=kwargs["litellm_params"],
kwargs=kwargs,
)
response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs)
assert json.loads(request_result)["messages"] == [{"role": "user", "content": "Hello!"}]
assert response_result == "{}"
def test_get_spend_logs_metadata_guardrail_info_fallback_from_metadata():
"""
When standard_logging_payload is None (e.g. guardrail blocks before LLM call),
@ -4460,7 +4574,7 @@ ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = (
'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n',
'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},'
'"usage":{"output_tokens":4}}\n\n',
"event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
'event: message_stop\ndata: {"type":"message_stop"}\n\n',
)
@ -4498,9 +4612,7 @@ def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_
"""
logging_obj = _anthropic_messages_logging_obj(stream=False)
logged_response = logging_obj._handle_anthropic_messages_response_logging(
result=ANTHROPIC_MESSAGES_RESPONSE
)
logged_response = logging_obj._handle_anthropic_messages_response_logging(result=ANTHROPIC_MESSAGES_RESPONSE)
assert logged_response.id == "msg_01Lit6806NonStreaming"
assert (
@ -4576,9 +4688,7 @@ def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_prov
end_time=datetime.datetime.now(timezone.utc),
logging_obj=logging_obj,
)
assert logging_obj.model_call_details["complete_streaming_response"].id == (
"6806cafe-0000-4000-8000-000000000001"
)
assert logging_obj.model_call_details["complete_streaming_response"].id == ("6806cafe-0000-4000-8000-000000000001")
def test_spend_log_request_id_for_chat_completions_is_untouched():

7
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -16781,7 +16781,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
@ -16887,7 +16886,6 @@ export interface paths {
* - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking.
* - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" }
* - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x.
* - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests.
* - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys)
* - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}.
* - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys)
@ -26163,6 +26161,11 @@ export interface components {
* @description If True, stores request messages and responses in spend logs. Default is False.
*/
store_prompts_in_spend_logs?: boolean | null;
/**
* Store Responses In Spend Logs
* @description Controls response content storage in spend logs independently. When unset, follows store_prompts_in_spend_logs for backward compatibility.
*/
store_responses_in_spend_logs?: boolean | null;
/**
* Supported Db Objects
* @description Fine-grained control over which object types to load from the database when store_model_in_db is True. Available types: 'models', 'mcp', 'guardrails', 'vector_stores', 'pass_through_endpoints', 'prompts', 'model_cost_map', 'tools', 'config_overrides'. If not set, all objects are loaded (default behavior).