scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(spend-logs): add truncation note when error logs are truncated for DB storage (#22936)

Browse source 
When the messages or response JSON fields in spend logs are truncated
before being written to the database, the truncation marker now includes
a note explaining:
- This is a DB storage safeguard
- Full, untruncated data is still sent to logging callbacks (OTEL, Datadog, etc.)
- The MAX_STRING_LENGTH_PROMPT_IN_DB env var can be used to increase the limit

Also emits a verbose_proxy_logger.info message when truncation occurs in
the request body or response spend log paths.

Adds 3 new tests:
- test_truncation_includes_db_safeguard_note
- test_response_truncation_logs_info_message
- test_request_body_truncation_logs_info_message

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
Krish Dholakia 2026-03-05 16:58:46 -08:00 • committed by GitHub
parent 92b3160206
commit 53a1e31729
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 110 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/constants.py
View file

@ -1242,6 +1242,11 @@ X_LITELLM_DISABLE_CALLBACKS = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD = "metadata"
LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = (
"Truncation is a DB storage safeguard. "
"Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.). "
"To increase the truncation limit, set `MAX_STRING_LENGTH_PROMPT_IN_DB` in your env."
)
########################### LiteLLM Proxy Specific Constants ###########################
########################################################################################

28
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -11,6 +11,10 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
from litellm.constants import \
MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB
from litellm.constants import REDACTED_BY_LITELM_STRING
@ -628,7 +632,10 @@ def _sanitize_request_body_for_spend_logs_payload(
Recursively sanitize request body to prevent logging large base64 strings or other large values.
Truncates strings longer than MAX_STRING_LENGTH_PROMPT_IN_DB characters and handles nested dictionaries.
"""
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
if visited is None:
visited = set()
@ -674,7 +681,8 @@ def _sanitize_request_body_for_spend_logs_payload(
# Build the truncated string: beginning + truncation marker + end
truncated_value = (
f"{value[:start_chars]}"
f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. "
f"{LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
f"{value[-end_chars:]}"
)
return truncated_value
@ -791,6 +799,11 @@ def _get_proxy_server_request_for_spend_logs_payload(
_request_body = _sanitize_request_body_for_spend_logs_payload(_request_body)
_request_body_json_str = json.dumps(_request_body, default=str)
if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str:
verbose_proxy_logger.info(
"Spend Log: request body was truncated before storing in DB. %s",
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
return _request_body_json_str
return "{}"
@ -866,8 +879,15 @@ def _get_response_for_spend_logs_payload(
if sanitized_response is None:
return "{}"
if isinstance(sanitized_response, str):
return sanitized_response
return safe_dumps(sanitized_response)
result_str = sanitized_response
else:
result_str = safe_dumps(sanitized_response)
if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str:
verbose_proxy_logger.info(
"Spend Log: response was truncated before storing in DB. %s",
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
return result_str
return "{}"

86
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -16,7 +16,11 @@ sys.path.insert(
from unittest.mock import AsyncMock, MagicMock, patch
import litellm
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, REDACTED_BY_LITELM_STRING
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
REDACTED_BY_LITELM_STRING,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_get_messages_for_spend_logs_payload,
@ -60,7 +64,7 @@ def test_sanitize_request_body_for_spend_logs_payload_long_string():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - (start_chars + end_chars)
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["text"]) == expected_length
@ -86,7 +90,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_dict():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["outer"]["inner"]["text"]) == expected_length
@ -111,7 +115,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_list():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["items"][0]["text"]) == expected_length
@ -151,7 +155,7 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars
assert len(sanitized["text"]) == expected_length
@ -396,6 +400,78 @@ def test_get_response_for_spend_logs_payload_truncates_large_embedding(mock_shou
assert parsed["data"][0]["other_field"] == "value"
def test_truncation_includes_db_safeguard_note():
"""
Test that truncated content includes the DB safeguard note explaining
that full data is available in OTEL/other logging integrations.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
large_error = "Error: " + "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 1000)
request_body = {"error_trace": large_error}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
truncated = sanitized["error_trace"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated
assert LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE in truncated
assert "DB storage safeguard" in truncated
assert "logging callbacks" in truncated.lower() or "logging integrations" in truncated.lower() or "logging callbacks" in truncated
@patch(
"litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
)
def test_response_truncation_logs_info_message(mock_should_store):
"""
Test that when response is truncated before DB storage, an info log is emitted
noting that full data is available in OTEL/other integrations.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
large_text = "B" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
payload = cast(
StandardLoggingPayload,
{"response": {"data": [{"content": large_text}]}},
)
with patch(
"litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger"
) as mock_logger:
_get_response_for_spend_logs_payload(payload)
mock_logger.info.assert_called_once()
log_msg = mock_logger.info.call_args[0][0]
assert "response was truncated" in log_msg
@patch(
"litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
)
def test_request_body_truncation_logs_info_message(mock_should_store):
"""
Test that when request body is truncated before DB storage, an info log is emitted.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
mock_should_store.return_value = True
large_prompt = "C" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
litellm_params = {
"proxy_server_request": {
"body": {"messages": [{"role": "user", "content": large_prompt}]}
}
}
with patch(
"litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger"
) as mock_logger:
_get_proxy_server_request_for_spend_logs_payload(
metadata={}, litellm_params=litellm_params, kwargs={}
)
mock_logger.info.assert_called_once()
log_msg = mock_logger.info.call_args[0][0]
assert "request body was truncated" in log_msg
def test_safe_dumps_handles_circular_references():
"""Test that safe_dumps can handle circular references without raising exceptions"""