From c935a816bfaf045aa722708c537ea9776a5b8af4 Mon Sep 17 00:00:00 2001 From: Gaurav Singh <103016722+gavksingh@users.noreply.github.com> Date: Thu, 26 Feb 2026 03:06:18 -0500 Subject: [PATCH] fix(proxy): honor MAX_STRING_LENGTH_PROMPT_IN_DB from config env vars (#22106) * fix(proxy): honor MAX_STRING_LENGTH_PROMPT_IN_DB from config env vars * fix(proxy): reuse constants fallback for MAX_STRING_LENGTH_PROMPT_IN_DB runtime resolver * test(proxy): restore PEP8 spacing between spend tracking tests --- .../spend_tracking/spend_tracking_utils.py | 39 +++++++++++++++---- .../test_spend_tracking_utils.py | 16 +++++++- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 1ef85562335..31615a768d7 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -1,5 +1,6 @@ import hashlib import json +import os import secrets from datetime import datetime from datetime import datetime as dt @@ -10,7 +11,10 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_proxy_logger -from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, REDACTED_BY_LITELM_STRING +from litellm.constants import ( + MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, +) +from litellm.constants import REDACTED_BY_LITELM_STRING from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, reconstruct_model_name, @@ -30,6 +34,20 @@ from litellm.types.utils import ( from litellm.utils import get_end_user_id_for_cost_tracking +def _get_max_string_length_prompt_in_db() -> int: + """ + Resolve prompt truncation threshold at runtime so values loaded later via + proxy config environment_variables are honored. + """ + max_length_str = os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB") + if max_length_str is None: + return DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB + try: + return int(max_length_str) + except (TypeError, ValueError): + return DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB + + def _is_master_key(api_key: str, _master_key: Optional[str]) -> bool: if _master_key is None: return False @@ -609,6 +627,7 @@ def _get_messages_for_spend_logs_payload( def _sanitize_request_body_for_spend_logs_payload( request_body: dict, visited: Optional[set] = None, + max_string_length_prompt_in_db: Optional[int] = None, ) -> dict: """ Recursively sanitize request body to prevent logging large base64 strings or other large values. @@ -618,6 +637,8 @@ def _sanitize_request_body_for_spend_logs_payload( if visited is None: visited = set() + if max_string_length_prompt_in_db is None: + max_string_length_prompt_in_db = _get_max_string_length_prompt_in_db() # Get the object's memory address to track visited objects obj_id = id(request_body) @@ -627,27 +648,29 @@ def _sanitize_request_body_for_spend_logs_payload( def _sanitize_value(value: Any) -> Any: if isinstance(value, dict): - return _sanitize_request_body_for_spend_logs_payload(value, visited) + return _sanitize_request_body_for_spend_logs_payload( + value, visited, max_string_length_prompt_in_db + ) elif isinstance(value, list): return [_sanitize_value(item) for item in value] elif isinstance(value, str): - if len(value) > MAX_STRING_LENGTH_PROMPT_IN_DB: + if len(value) > max_string_length_prompt_in_db: # Keep 35% from beginning and 65% from end (end is usually more important) # This split ensures we keep more context from the end of conversations start_ratio = 0.35 end_ratio = 0.65 # Calculate character distribution - start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * start_ratio) - end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * end_ratio) + start_chars = int(max_string_length_prompt_in_db * start_ratio) + end_chars = int(max_string_length_prompt_in_db * end_ratio) # Ensure we don't exceed the total limit total_keep = start_chars + end_chars - if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB: - end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars + if total_keep > max_string_length_prompt_in_db: + end_chars = max_string_length_prompt_in_db - start_chars # If the string length is less than what we want to keep, just truncate normally - if len(value) <= MAX_STRING_LENGTH_PROMPT_IN_DB: + if len(value) <= max_string_length_prompt_in_db: return value # Calculate how many characters are being skipped diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 30257cd2da8..24f45cc5c91 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -161,6 +161,21 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types(): assert len(sanitized["nested"]["dict"]["key"]) == expected_length +def test_sanitize_request_body_for_spend_logs_payload_uses_runtime_env_override( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + override_max = max(MAX_STRING_LENGTH_PROMPT_IN_DB + 1000, 6000) + test_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500) + + # Simulate config-loaded env var being set after module import. + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", str(override_max)) + + sanitized = _sanitize_request_body_for_spend_logs_payload({"text": test_string}) + assert sanitized["text"] == test_string + + def test_sanitize_request_body_for_spend_logs_payload_circular_reference(): # Create a circular reference a: dict[str, Any] = {} @@ -1349,4 +1364,3 @@ def test_get_logging_payload_includes_request_duration_ms(): ) assert payload["request_duration_ms"] == 3000 -