scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

feat(spend-logs): add truncation note when error logs are truncated for DB storage

Browse source 
When the messages or response JSON fields in spend logs are truncated
before being written to the database, the truncation marker now includes
a note explaining:
- This is a DB storage safeguard
- Full, untruncated data is still sent to logging callbacks (OTEL, Datadog, etc.)
- The MAX_STRING_LENGTH_PROMPT_IN_DB env var can be used to increase the limit

Also emits a verbose_proxy_logger.info message when truncation occurs in
the request body or response spend log paths.

Adds 3 new tests:
- test_truncation_includes_db_safeguard_note
- test_response_truncation_logs_info_message
- test_request_body_truncation_logs_info_message

Co-authored-by: Krish Dholakia <krrishdholakia@gmail.com>
This commit is contained in:
Cursor Agent 2026-03-05 23:43:44 +00:00
parent 3d027c0f7a
commit b21140775d
3 changed files with 110 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/constants.py
View file

@ -1242,6 +1242,11 @@ X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD = "metadata"
LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = (
"Truncation is a DB storage safeguard. "
"Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.). "
"To increase the truncation limit, set `MAX_STRING_LENGTH_PROMPT_IN_DB` in your env."
)
########################### LiteLLM Proxy Specific Constants ###########################
########################################################################################

28
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -11,6 +11,10 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
from litellm.constants import \
MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB
from litellm.constants import REDACTED_BY_LITELM_STRING
@ -628,7 +632,10 @@ def _sanitize_request_body_for_spend_logs_payload(
Recursively sanitize request body to prevent logging large base64 strings or other large values.
Truncates strings longer than MAX_STRING_LENGTH_PROMPT_IN_DB characters and handles nested dictionaries.
"""
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
if visited is None:
visited = set()
@ -674,7 +681,8 @@ def _sanitize_request_body_for_spend_logs_payload(
# Build the truncated string: beginning + truncation marker + end
truncated_value = (
f"{value[:start_chars]}"
f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. "
f"{LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
f"{value[-end_chars:]}"
)
return truncated_value
@ -791,6 +799,11 @@ def _get_proxy_server_request_for_spend_logs_payload(
_request_body = _sanitize_request_body_for_spend_logs_payload(_request_body)
_request_body_json_str = json.dumps(_request_body, default=str)
if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str:
verbose_proxy_logger.info(
"Spend Log: request body was truncated before storing in DB. %s",
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
return _request_body_json_str
return "{}"
@ -866,8 +879,15 @@ def _get_response_for_spend_logs_payload(
if sanitized_response is None:
return "{}"
if isinstance(sanitized_response, str):
return sanitized_response
return safe_dumps(sanitized_response)
result_str = sanitized_response
else:
result_str = safe_dumps(sanitized_response)
if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str:
verbose_proxy_logger.info(
"Spend Log: response was truncated before storing in DB. %s",
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
return result_str
return "{}"

86
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -16,7 +16,11 @@ sys.path.insert(
from unittest.mock import AsyncMock, MagicMock, patch
import litellm
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, REDACTED_BY_LITELM_STRING
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
REDACTED_BY_LITELM_STRING,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_get_messages_for_spend_logs_payload,
@ -60,7 +64,7 @@ def test_sanitize_request_body_for_spend_logs_payload_long_string():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - (start_chars + end_chars)
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["text"]) == expected_length
@ -86,7 +90,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_dict():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["outer"]["inner"]["text"]) == expected_length
@ -111,7 +115,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_list():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["items"][0]["text"]) == expected_length
@ -151,7 +155,7 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["text"]) == expected_length
@ -396,6 +400,78 @@ def test_get_response_for_spend_logs_payload_truncates_large_embedding(mock_shou
assert parsed["data"][0]["other_field"] == "value"
def test_truncation_includes_db_safeguard_note():
"""
Test that truncated content includes the DB safeguard note explaining
that full data is available in OTEL/other logging integrations.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
large_error = "Error: " + "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 1000)
request_body = {"error_trace": large_error}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
truncated = sanitized["error_trace"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated
assert LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE in truncated
assert "DB storage safeguard" in truncated
assert "logging callbacks" in truncated.lower() or "logging integrations" in truncated.lower() or "logging callbacks" in truncated
@patch(
"litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
)
def test_response_truncation_logs_info_message(mock_should_store):
"""
Test that when response is truncated before DB storage, an info log is emitted
noting that full data is available in OTEL/other integrations.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
large_text = "B" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
payload = cast(
StandardLoggingPayload,
{"response": {"data": [{"content": large_text}]}},
)
with patch(
"litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger"
) as mock_logger:
_get_response_for_spend_logs_payload(payload)
mock_logger.info.assert_called_once()
log_msg = mock_logger.info.call_args[0][0]
assert "response was truncated" in log_msg
@patch(
"litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
)
def test_request_body_truncation_logs_info_message(mock_should_store):
"""
Test that when request body is truncated before DB storage, an info log is emitted.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
large_prompt = "C" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
litellm_params = {
"proxy_server_request": {
"body": {"messages": [{"role": "user", "content": large_prompt}]}
}
}
with patch(
"litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger"
) as mock_logger:
_get_proxy_server_request_for_spend_logs_payload(
metadata={}, litellm_params=litellm_params, kwargs={}
)
mock_logger.info.assert_called_once()
log_msg = mock_logger.info.call_args[0][0]
assert "request body was truncated" in log_msg
def test_safe_dumps_handles_circular_references():
"""Test that safe_dumps can handle circular references without raising exceptions"""