diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b315a2beac9..db4c7f1c874 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2754,6 +2754,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="If True, stores request messages and responses in spend logs. Default is False.", ) + store_responses_in_spend_logs: bool | None = Field( + None, + description="Controls response content storage in spend logs independently. When unset, follows store_prompts_in_spend_logs for backward compatibility.", + ) disable_auto_add_proxy_admin_to_teams: bool | None = Field( None, description="By default, the user calling /team/new is automatically added to the new team as a team admin. If True, proxy admins are no longer auto-added; members explicitly listed in members_with_roles are unaffected. Default is False.", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d9f8e04ebda..a7942f13ea2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7107,6 +7107,21 @@ class ProxyConfig: # For other types, convert to bool general_settings["store_prompts_in_spend_logs"] = bool(value) + if "store_responses_in_spend_logs" in _general_settings: + response_storage_value: Final = ( + general_settings.get("store_responses_in_spend_logs") + if "store_responses_in_spend_logs" in self._yaml_general_settings_keys + else _general_settings["store_responses_in_spend_logs"] + ) + if response_storage_value is None: + general_settings["store_responses_in_spend_logs"] = None + elif isinstance(response_storage_value, bool): + general_settings["store_responses_in_spend_logs"] = response_storage_value + elif isinstance(response_storage_value, str): + general_settings["store_responses_in_spend_logs"] = response_storage_value.lower() == "true" + else: + general_settings["store_responses_in_spend_logs"] = bool(response_storage_value) + if "disable_auto_add_proxy_admin_to_teams" in _general_settings: value = _general_settings["disable_auto_add_proxy_admin_to_teams"] if isinstance(value, str): @@ -17042,6 +17057,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro "pass_through_endpoints": "PydanticModel", "store_model_in_db": "Boolean", "store_prompts_in_spend_logs": "Boolean", + "store_responses_in_spend_logs": "Boolean", "maximum_spend_logs_retention_period": "String", "maximum_health_check_retention_period": "String", "maximum_spend_logs_cleanup_batch_size": "Integer", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 3ef21996b9c..7c8feb7c968 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -53,7 +53,6 @@ from litellm.types.utils import ( StandardLoggingPayload, StandardLoggingPayloadErrorInformation, StandardLoggingVectorStoreRequest, - VectorStoreSearchResponse, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -199,14 +198,19 @@ def _get_spend_logs_metadata( _already_redacted: Final = ( isinstance(_trusted_hash, str) and _is_non_secret_key_value(_trusted_hash) and _trusted_hash == _raw_key ) + store_responses: Final = _should_store_responses_in_spend_logs() clean_metadata["user_api_key"] = _redact_logged_api_key(_raw_key, already_redacted=_already_redacted) clean_metadata["applied_guardrails"] = applied_guardrails clean_metadata["batch_models"] = batch_models clean_metadata["batch_successful_requests"] = batch_successful_requests clean_metadata["batch_failed_requests"] = batch_failed_requests - clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata + clean_metadata["mcp_tool_call_metadata"] = _get_mcp_tool_call_metadata_for_spend_logs_payload( + mcp_tool_call_metadata, + store_responses=store_responses, + ) clean_metadata["vector_store_request_metadata"] = _get_vector_store_request_for_spend_logs_payload( - vector_store_request_metadata + vector_store_request_metadata, + store_responses=store_responses, ) clean_metadata["guardrail_information"] = _sanitize_guardrail_information_for_spend_logs(guardrail_information) clean_metadata["usage_object"] = usage_object @@ -1322,28 +1326,36 @@ def _get_proxy_server_request_for_spend_logs_payload( return "{}" +def _get_mcp_tool_call_metadata_for_spend_logs_payload( + mcp_tool_call_metadata: StandardLoggingMCPToolCall | None, + *, + store_responses: bool, +) -> StandardLoggingMCPToolCall | None: + if mcp_tool_call_metadata is None or store_responses: + return mcp_tool_call_metadata + return cast( + StandardLoggingMCPToolCall, + {key: value for key, value in mcp_tool_call_metadata.items() if key != "result"}, + ) + + def _get_vector_store_request_for_spend_logs_payload( vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None, + *, + store_responses: bool, ) -> list[StandardLoggingVectorStoreRequest] | None: - """ - If user does not want to store prompts and responses, then remove the content from the vector store request metadata - """ - if should_store_prompts_and_responses_in_spend_logs(): + if store_responses: return vector_store_request_metadata - # if user does not want to store prompts and responses, then remove the content from the vector store request metadata if vector_store_request_metadata is None: return None - for vector_store_request in vector_store_request_metadata: - vector_store_search_response: VectorStoreSearchResponse = ( - vector_store_request.get("vector_store_search_response") or VectorStoreSearchResponse() + return [ + cast( + StandardLoggingVectorStoreRequest, + {key: value for key, value in vector_store_request.items() if key != "vector_store_search_response"}, ) - response_data = vector_store_search_response.get("data", []) or [] - for response_item in response_data: - for content_item in response_item.get("content", []) or []: - if "text" in content_item: - content_item["text"] = REDACTED_BY_LITELM_STRING - return vector_store_request_metadata + for vector_store_request in vector_store_request_metadata + ] def _get_response_for_spend_logs_payload( @@ -1352,7 +1364,7 @@ def _get_response_for_spend_logs_payload( ) -> str: if payload is None: return "{}" - if should_store_prompts_and_responses_in_spend_logs(): + if _should_store_responses_in_spend_logs(): response_obj: object = payload.get("response") if response_obj is None: return "{}" @@ -1419,6 +1431,18 @@ def should_store_prompts_and_responses_in_spend_logs() -> bool: return get_secret_bool("STORE_PROMPTS_IN_SPEND_LOGS") is True +def _should_store_responses_in_spend_logs() -> bool: + from litellm.proxy.proxy_server import general_settings + + store_responses_value: Final = general_settings.get("store_responses_in_spend_logs") + if store_responses_value is not None: + if isinstance(store_responses_value, str): + return store_responses_value.lower() == "true" + return store_responses_value is True + + return should_store_prompts_and_responses_in_spend_logs() + + def _get_status_for_spend_log( metadata: dict, ) -> Literal["success", "failure"]: diff --git a/tests/proxy_unit_tests/test_proxy_config_unit_test.py b/tests/proxy_unit_tests/test_proxy_config_unit_test.py index 81648dc1158..ce05b769ad6 100644 --- a/tests/proxy_unit_tests/test_proxy_config_unit_test.py +++ b/tests/proxy_unit_tests/test_proxy_config_unit_test.py @@ -353,3 +353,44 @@ class TestYamlStorePromptsDbOverride: """_yaml_general_settings_keys should be empty on init.""" proxy_config = ProxyConfig() assert proxy_config._yaml_general_settings_keys == set() + + @pytest.mark.asyncio + async def test_response_storage_yaml_value_takes_precedence_over_db(self): + proxy_config = self._make_proxy_config_with_yaml_keys({"store_responses_in_spend_logs"}) + test_general_settings = {"store_responses_in_spend_logs": False} + + with mock.patch( # test-quality-ok: verifies synchronization with the process-wide settings store + "litellm.proxy.proxy_server.general_settings", test_general_settings + ): + await proxy_config._update_general_settings( + db_general_settings={"store_responses_in_spend_logs": True}, + ) + + assert test_general_settings["store_responses_in_spend_logs"] is False + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("db_value", "expected"), + [ + (None, None), + (True, True), + ("TRUE", True), + (0, False), + ], + ) + async def test_response_storage_db_value_is_normalized_when_yaml_omits_key( + self, + db_value, + expected, + ): + proxy_config = self._make_proxy_config_with_yaml_keys({"master_key"}) + test_general_settings = {"master_key": "sk-test"} + + with mock.patch( # test-quality-ok: verifies synchronization with the process-wide settings store + "litellm.proxy.proxy_server.general_settings", test_general_settings + ): + await proxy_config._update_general_settings( + db_general_settings={"store_responses_in_spend_logs": db_value}, + ) + + assert test_general_settings["store_responses_in_spend_logs"] is expected diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index a72b4e28143..cb1d869a33b 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -27,6 +27,7 @@ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_messages_for_spend_logs_payload, + _get_mcp_tool_call_metadata_for_spend_logs_payload, _get_proxy_server_request_for_spend_logs_payload, _get_request_duration_ms, _get_response_for_spend_logs_payload, @@ -39,6 +40,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( _sanitize_error_information_for_spend_logs, _sanitize_guardrail_information_for_spend_logs, _sanitize_request_body_for_spend_logs_payload, + _should_store_responses_in_spend_logs, get_logging_payload, get_spend_logs_id, should_store_prompts_and_responses_in_spend_logs, @@ -46,9 +48,11 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( from litellm.proxy.utils import hash_token from litellm.types.utils import ( StandardLoggingHiddenParams, + StandardLoggingMCPToolCall, StandardLoggingMetadata, StandardLoggingModelInformation, StandardLoggingPayload, + StandardLoggingVectorStoreRequest, ) @@ -79,10 +83,16 @@ def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch "classifier_input": {"system": "rubric" * 1000, "messages": [{"role": "user", "content": "ask"}]}, "originating_request_masked": {"input": "source-only", "api_key": "REDACTED"}, } - stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload( - metadata={}, litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}}, - kwargs={"standard_logging_object": audit, "standard_callback_dynamic_params": {"turn_off_message_logging": redact}}, - )) + stored: Final = json.loads( + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, + litellm_params={"proxy_server_request": {"body": {"model": "classifier"}}}, + kwargs={ + "standard_logging_object": audit, + "standard_callback_dynamic_params": {"turn_off_message_logging": redact}, + }, + ) + ) if not store_prompts or redact: assert "classifier_input" not in stored assert "originating_request_masked" not in stored @@ -208,9 +218,7 @@ def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id(): from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1") - cost_session: Final = _get_batch_trace_session_id( - call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost" - ) + cost_session: Final = _get_batch_trace_session_id(call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost") assert create_session == cost_session == "batch-uid-1" @@ -656,52 +664,86 @@ def test_sanitize_request_body_for_spend_logs_payload_circular_reference(): assert sanitized == {"b": {"a": {}}} # Should return empty dict for circular reference -@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") -def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true( - mock_should_store, +@pytest.mark.parametrize( + ("store_responses", "expected_response_stored"), + [ + (False, False), + (True, True), + ], +) +def test_get_vector_store_request_for_spend_logs_payload_uses_response_setting( + store_responses: bool, expected_response_stored: bool ): - # When should_store_prompts_and_responses_in_spend_logs returns True - mock_should_store.return_value = True - - # Sample vector store request metadata - vector_store_request = [ - {"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}} + vector_store_request: Final[list[StandardLoggingVectorStoreRequest]] = [ + { + "vector_store_id": "vs-123", + "custom_llm_provider": "openai", + "query": "request content", + "start_time": 1.0, + "end_time": 2.0, + "vector_store_search_response": { + "search_query": "request content", + "data": [ + { + "file_id": "file-123", + "filename": "sensitive-filename.txt", + "attributes": {"source_url": "https://sensitive.example"}, + "content": [{"text": "sensitive information", "type": "text"}], + } + ], + }, + } ] - # When store_prompts is True, the original data should be returned unchanged - result = _get_vector_store_request_for_spend_logs_payload(vector_store_request) - assert result == vector_store_request - assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == "sensitive information" + result: Final = _get_vector_store_request_for_spend_logs_payload( + vector_store_request, + store_responses=store_responses, + ) - -@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") -def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false( - mock_should_store, -): - # When should_store_prompts_and_responses_in_spend_logs returns False - mock_should_store.return_value = False - - # Sample vector store request metadata - vector_store_request = [ - {"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}} - ] - - # When store_prompts is False, text should be redacted - result = _get_vector_store_request_for_spend_logs_payload(vector_store_request) assert result is not None - assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == REDACTED_BY_LITELM_STRING - # Ensure other fields are unchanged - assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"] == "text" + assert result[0]["vector_store_id"] == "vs-123" + assert result[0]["custom_llm_provider"] == "openai" + assert result[0]["query"] == "request content" + assert result[0]["start_time"] == 1.0 + assert result[0]["end_time"] == 2.0 + expected_response: Final = ( + vector_store_request[0]["vector_store_search_response"] if expected_response_stored else None + ) + assert result[0].get("vector_store_search_response") == expected_response + assert ("vector_store_search_response" in result[0]) is expected_response_stored -@patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") -def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_store): - # When input is None - mock_should_store.return_value = False - result = _get_vector_store_request_for_spend_logs_payload(None) +def test_get_vector_store_request_for_spend_logs_payload_null_input(): + result = _get_vector_store_request_for_spend_logs_payload(None, store_responses=False) assert result is None +@pytest.mark.parametrize( + ("store_responses", "expected_result"), + [ + (False, None), + (True, {"content": "sensitive response"}), + ], +) +def test_get_spend_logs_metadata_uses_response_setting_for_mcp_result( + store_responses: bool, expected_result: Mapping[str, str] | None +): + mcp_metadata: Final[StandardLoggingMCPToolCall] = { + "name": "search", + "arguments": {"query": "request content"}, + "result": {"content": "sensitive response"}, + } + + stored_mcp_metadata: Final = _get_mcp_tool_call_metadata_for_spend_logs_payload( + mcp_metadata, + store_responses=store_responses, + ) + assert stored_mcp_metadata is not None + assert stored_mcp_metadata["name"] == "search" + assert stored_mcp_metadata["arguments"] == {"query": "request content"} + assert stored_mcp_metadata.get("result") == expected_result + + @patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store): """ @@ -1708,6 +1750,78 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin assert result is False, "Expected False (from env var) when key missing, got True" +@pytest.mark.parametrize( + ("settings", "expected"), + [ + ({"store_prompts_in_spend_logs": True}, True), + ({"store_prompts_in_spend_logs": False}, False), + ( + { + "store_prompts_in_spend_logs": True, + "store_responses_in_spend_logs": False, + }, + False, + ), + ( + { + "store_prompts_in_spend_logs": False, + "store_responses_in_spend_logs": True, + }, + True, + ), + ( + { + "store_prompts_in_spend_logs": True, + "store_responses_in_spend_logs": "FALSE", + }, + False, + ), + ], +) +def test_should_store_responses_in_spend_logs( + settings, + expected, +): + with patch( # test-quality-ok: isolates the process-wide settings source for retention resolution + "litellm.proxy.proxy_server.general_settings", settings + ): + assert _should_store_responses_in_spend_logs() is expected + + +def test_spend_logs_can_store_request_without_response(): + settings = { + "store_prompts_in_spend_logs": True, + "store_responses_in_spend_logs": False, + } + kwargs = { + "litellm_params": { + "proxy_server_request": { + "body": { + "model": "gpt-5-mini", + "messages": [{"role": "user", "content": "Hello!"}], + } + } + } + } + payload = cast( + StandardLoggingPayload, + {"response": {"role": "assistant", "content": "Hi there!"}}, + ) + + with patch( # test-quality-ok: isolates the process-wide settings source for the retention integration check + "litellm.proxy.proxy_server.general_settings", settings + ): + request_result = _get_proxy_server_request_for_spend_logs_payload( + metadata={}, + litellm_params=kwargs["litellm_params"], + kwargs=kwargs, + ) + response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs) + + assert json.loads(request_result)["messages"] == [{"role": "user", "content": "Hello!"}] + assert response_result == "{}" + + def test_get_spend_logs_metadata_guardrail_info_fallback_from_metadata(): """ When standard_logging_payload is None (e.g. guardrail blocks before LLM call), @@ -4460,7 +4574,7 @@ ANTHROPIC_MESSAGES_SSE_CHUNKS: Final = ( 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn"},' '"usage":{"output_tokens":4}}\n\n', - "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n", + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', ) @@ -4498,9 +4612,7 @@ def test_spend_log_request_id_is_the_message_id_a_non_streaming_messages_caller_ """ logging_obj = _anthropic_messages_logging_obj(stream=False) - logged_response = logging_obj._handle_anthropic_messages_response_logging( - result=ANTHROPIC_MESSAGES_RESPONSE - ) + logged_response = logging_obj._handle_anthropic_messages_response_logging(result=ANTHROPIC_MESSAGES_RESPONSE) assert logged_response.id == "msg_01Lit6806NonStreaming" assert ( @@ -4576,9 +4688,7 @@ def test_spend_log_request_id_still_falls_back_to_litellm_call_id_without_a_prov end_time=datetime.datetime.now(timezone.utc), logging_obj=logging_obj, ) - assert logging_obj.model_call_details["complete_streaming_response"].id == ( - "6806cafe-0000-4000-8000-000000000001" - ) + assert logging_obj.model_call_details["complete_streaming_response"].id == ("6806cafe-0000-4000-8000-000000000001") def test_spend_log_request_id_for_chat_completions_is_untouched(): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7eadaa6c991..0e020ef170c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16781,7 +16781,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -16887,7 +16886,6 @@ export interface paths { * - permissions: Optional[dict] - [Not Implemented Yet] User-specific permissions, eg. turning off pii masking. * - metadata: Optional[dict] - Metadata for user, store information for user. Example metadata = {"team": "core-infra", "app": "app2", "email": "ishaan@berri.ai" } * - max_parallel_requests: Optional[int] - Rate limit a user based on the number of parallel requests. Raises 429 error, if user's parallel requests > x. - * - soft_budget: Optional[float] - Get alerts when user crosses given budget, doesn't block requests. * - model_max_budget: Optional[dict] - Model-specific max budget for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-budgets-to-keys) * - budget_fallbacks: Optional[Dict[str, List[str]]] - Per-model fallback chain tried in order when that model's own `model_max_budget` is exceeded, e.g. {"gpt-4o": ["gpt-4o-mini"]}. * - model_rpm_limit: Optional[float] - Model-specific rpm limit for user. [Docs](https://docs.litellm.ai/docs/proxy/users#add-model-specific-limits-to-keys) @@ -26163,6 +26161,11 @@ export interface components { * @description If True, stores request messages and responses in spend logs. Default is False. */ store_prompts_in_spend_logs?: boolean | null; + /** + * Store Responses In Spend Logs + * @description Controls response content storage in spend logs independently. When unset, follows store_prompts_in_spend_logs for backward compatibility. + */ + store_responses_in_spend_logs?: boolean | null; /** * Supported Db Objects * @description Fine-grained control over which object types to load from the database when store_model_in_db is True. Available types: 'models', 'mcp', 'guardrails', 'vector_stores', 'pass_through_endpoints', 'prompts', 'model_cost_map', 'tools', 'config_overrides'. If not set, all objects are loaded (default behavior).