From 1419fde0dead8adfc5835b17ed66423cb9f75552 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 5 Jan 2026 05:45:48 +0900 Subject: [PATCH] feat: default spend logs to proxy-modified responses --- docs/my-website/docs/proxy/ui_logs.md | 12 +- .../proxy/hooks/proxy_track_cost_callback.py | 4 +- litellm/proxy/proxy_server.py | 22 +--- .../hooks/test_proxy_track_cost_callback.py | 105 ++++++++++-------- 4 files changed, 72 insertions(+), 71 deletions(-) diff --git a/docs/my-website/docs/proxy/ui_logs.md b/docs/my-website/docs/proxy/ui_logs.md index c77399b494d..d4cb85d5514 100644 --- a/docs/my-website/docs/proxy/ui_logs.md +++ b/docs/my-website/docs/proxy/ui_logs.md @@ -55,14 +55,20 @@ general_settings: ## Store Proxy-Modified Responses in Spend Logs -By default the spend log records the upstream LLM response (before any proxy guardrails or rewrites). Opt in to storing the proxy-mutated response with: +By default the spend log records the proxy-modified response (after any proxy guardrails or rewrites). If you prefer to log the upstream LLM response, disable the proxy mutation storage: ```yaml general_settings: - spend_logs_store_proxy_response: true + spend_logs_store_proxy_response: false ``` -When set to `true`, UI logs show the final post-guardrail response seen by end users. Set to `false` to keep logging the raw LLM response. +When set to `false`, UI logs show the raw LLM response seen before proxy guardrails. Keep this value `true` (the default) to continue logging the final post-guardrail response. + +:::note Change in v1.80.1 + +Starting in v1.80.1, proxy-modified responses are stored in spend logs by default. Set `spend_logs_store_proxy_response: false` to restore the previous behavior of logging the upstream LLM response. + +::: ## Automatically Deleting Old Spend Logs diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index daf55a12590..c5e65c3d062 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -36,7 +36,7 @@ class _ProxyDBLogger(CustomLogger): ): """Persist spend logs after proxy guardrails mutate metadata.""" - if ProxyUpdateSpend.should_store_proxy_response_in_spend_logs() is not True: + if ProxyUpdateSpend.should_store_proxy_response_in_spend_logs() is False: return logging_obj = data.get("litellm_logging_obj", None) @@ -219,7 +219,7 @@ class _ProxyDBLogger(CustomLogger): ## UPDATE DATABASE if ( ProxyUpdateSpend.should_store_proxy_response_in_spend_logs() - is not True + is False ): await proxy_logging_obj.db_spend_update_writer.update_database( token=user_api_key, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 75fac7d37a9..756e5ec312d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4,7 +4,6 @@ import inspect import io import os import random -import secrets import shutil import subprocess import sys @@ -26,7 +25,6 @@ from typing import ( cast, get_args, get_origin, - get_type_hints, ) from litellm._uuid import uuid @@ -34,7 +32,6 @@ from litellm.constants import ( AIOHTTP_CONNECTOR_LIMIT, AIOHTTP_CONNECTOR_LIMIT_PER_HOST, AIOHTTP_KEEPALIVE_TIMEOUT, - AIOHTTP_NEEDS_CLEANUP_CLOSED, AIOHTTP_TTL_DNS_CACHE, AUDIO_SPEECH_CHUNK_SIZE, BASE_MCP_ROUTE, @@ -54,7 +51,6 @@ from litellm.proxy.common_utils.realtime_utils import _realtime_request_body from litellm.types.utils import ( ModelResponse, ModelResponseStream, - TextCompletionResponse, TokenCountResponse, ) from litellm.utils import load_credentials_from_list @@ -1320,22 +1316,12 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False): def cost_tracking(): global prisma_client if prisma_client is None: - return - - store_proxy_response = ProxyUpdateSpend.should_store_proxy_response_in_spend_logs() - disable_spend = ProxyUpdateSpend.disable_spend_updates() - - if store_proxy_response is None and not disable_spend: - verbose_proxy_logger.warning( - "general_settings.spend_logs_store_proxy_response is not set. " - "Current default logs the upstream LLM response; this will change in a future release. " - "Set it to True to store the proxy-mutated response or False to keep current behavior." + proxy_db_logger = _ProxyDBLogger() + litellm.logging_callback_manager.add_litellm_callback(proxy_db_logger) + litellm.logging_callback_manager.add_litellm_async_success_callback( + proxy_db_logger ) - proxy_db_logger = _ProxyDBLogger() - litellm.logging_callback_manager.add_litellm_callback(proxy_db_logger) - litellm.logging_callback_manager.add_litellm_async_success_callback(proxy_db_logger) - async def update_cache( # noqa: PLR0915 token: Optional[str], diff --git a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py index 373a604d410..29ffb26586a 100644 --- a/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py @@ -27,10 +27,16 @@ def mock_proxy_logging_obj(monkeypatch): def __init__(self): self.update_database = AsyncMock() + class _ServiceLoggingStub: + def __init__(self): + self.async_service_success_hook = AsyncMock() + self.async_service_failure_hook = AsyncMock() + class _ProxyLoggingStub: def __init__(self): self.db_spend_update_writer = _DBWriterStub() self.slack_alerting_instance = _SlackStub() + self.service_logging_obj = _ServiceLoggingStub() stub = _ProxyLoggingStub() monkeypatch.setattr( @@ -47,7 +53,7 @@ def mock_proxy_logging_obj(monkeypatch): @pytest.mark.asyncio -async def test_async_post_call_failure_hook(): +async def test_async_post_call_failure_hook(mock_proxy_logging_obj): # Setup logger = _ProxyDBLogger() @@ -74,42 +80,39 @@ async def test_async_post_call_failure_hook(): # Mock exception original_exception = Exception("Test exception") - # Mock update_database function - with patch( - "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", - new_callable=AsyncMock, - ) as mock_update_database: - # Call the method - await logger.async_post_call_failure_hook( - request_data=request_data, - original_exception=original_exception, - user_api_key_dict=user_api_key_dict, - ) + # Call the method + await logger.async_post_call_failure_hook( + request_data=request_data, + original_exception=original_exception, + user_api_key_dict=user_api_key_dict, + ) - # Assertions - mock_update_database.assert_called_once() + update_database = mock_proxy_logging_obj.db_spend_update_writer.update_database - # Check the arguments passed to update_database - call_args = mock_update_database.call_args[1] - print("call_args", json.dumps(call_args, indent=4, default=str)) - assert call_args["token"] == "test_api_key" - assert call_args["response_cost"] == 0.0 - assert call_args["user_id"] == "test_user_id" - assert call_args["end_user_id"] == "test_end_user_id" - assert call_args["team_id"] == "test_team_id" - assert call_args["org_id"] == "test_org_id" - assert call_args["completion_response"] == original_exception + # Assertions + update_database.assert_awaited_once() - # Check that metadata was properly updated - assert "litellm_params" in call_args["kwargs"] - assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == { - "request_id": "test_request_id" - } - metadata = call_args["kwargs"]["litellm_params"]["metadata"] - assert metadata["user_api_key"] == "test_api_key" - assert metadata["status"] == "failure" - assert "error_information" in metadata - assert metadata["original_key"] == "original_value" + # Check the arguments passed to update_database + call_args = update_database.await_args.kwargs + print("call_args", json.dumps(call_args, indent=4, default=str)) + assert call_args["token"] == "test_api_key" + assert call_args["response_cost"] == 0.0 + assert call_args["user_id"] == "test_user_id" + assert call_args["end_user_id"] == "test_end_user_id" + assert call_args["team_id"] == "test_team_id" + assert call_args["org_id"] == "test_org_id" + assert call_args["completion_response"] == original_exception + + # Check that metadata was properly updated + assert "litellm_params" in call_args["kwargs"] + assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == { + "request_id": "test_request_id" + } + metadata = call_args["kwargs"]["litellm_params"]["metadata"] + assert metadata["user_api_key"] == "test_api_key" + assert metadata["status"] == "failure" + assert "error_information" in metadata + assert metadata["original_key"] == "original_value" @pytest.mark.asyncio @@ -161,15 +164,17 @@ async def test_async_post_call_failure_hook_non_llm_route(): async def test_async_log_success_event_writes_db_when_flag_false(monkeypatch, mock_proxy_logging_obj): logger = _ProxyDBLogger() + metadata = { + "user_api_key": "sk-test", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "team-1", + "user_api_key_org_id": "org-1", + } + kwargs = { "model": "gpt-4", - "metadata": { - "user_api_key": "sk-test", - "user_api_key_user_id": "user-1", - "user_api_key_team_id": "team-1", - "user_api_key_org_id": "org-1", - }, - "litellm_params": {}, + "metadata": metadata, + "litellm_params": {"metadata": metadata.copy()}, "standard_logging_object": StandardLoggingPayload( id="req-1", trace_id=None, @@ -359,7 +364,7 @@ async def test_async_log_success_event_with_flag_none(monkeypatch, mock_proxy_lo end_time=datetime.now(), ) - mock_proxy_logging_obj.db_spend_update_writer.update_database.assert_awaited_once() + mock_proxy_logging_obj.db_spend_update_writer.update_database.assert_not_called() @pytest.mark.asyncio @@ -370,14 +375,18 @@ async def test_async_post_call_success_hook_writes_db_with_guardrail_info( class _LoggingObj: def __init__(self): + metadata = { + "user_api_key": "sk-test", + "user_api_key_user_id": "user-1", + "user_api_key_team_id": "team-1", + "user_api_key_org_id": "org-1", + } self.model_call_details = { - "metadata": { - "user_api_key": "sk-test", - "user_api_key_user_id": "user-1", - "user_api_key_team_id": "team-1", - "user_api_key_org_id": "org-1", + "metadata": metadata, + "litellm_params": { + "metadata": metadata.copy(), + "litellm_metadata": metadata.copy(), }, - "litellm_params": {}, "standard_logging_object": { "response_cost": 0.25, },