scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 227

feat: default spend logs to proxy-modified responses

Browse source
This commit is contained in:
Yuta Saito 2026-01-05 05:45:48 +09:00
parent 841ca956c0
commit 1419fde0de
4 changed files with 72 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
docs/my-website/docs/proxy/ui_logs.md
View file

@ -55,14 +55,20 @@ general_settings:
## Store Proxy-Modified Responses in Spend Logs
By default the spend log records the upstream LLM response (before any proxy guardrails or rewrites). Opt in to storing the proxy-mutated response with:
By default the spend log records the proxy-modified response (after any proxy guardrails or rewrites). If you prefer to log the upstream LLM response, disable the proxy mutation storage:
```yaml
general_settings:
spend_logs_store_proxy_response: true
spend_logs_store_proxy_response: false
```
When set to `true`, UI logs show the final post-guardrail response seen by end users. Set to `false` to keep logging the raw LLM response.
When set to `false`, UI logs show the raw LLM response seen before proxy guardrails. Keep this value `true` (the default) to continue logging the final post-guardrail response.
:::note Change in v1.80.1
Starting in v1.80.1, proxy-modified responses are stored in spend logs by default. Set `spend_logs_store_proxy_response: false` to restore the previous behavior of logging the upstream LLM response.
:::
## Automatically Deleting Old Spend Logs

4
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -36,7 +36,7 @@ class _ProxyDBLogger(CustomLogger):
):
"""Persist spend logs after proxy guardrails mutate metadata."""
if ProxyUpdateSpend.should_store_proxy_response_in_spend_logs() is not True:
if ProxyUpdateSpend.should_store_proxy_response_in_spend_logs() is False:
return
logging_obj = data.get("litellm_logging_obj", None)
@ -219,7 +219,7 @@ class _ProxyDBLogger(CustomLogger):
## UPDATE DATABASE
if (
ProxyUpdateSpend.should_store_proxy_response_in_spend_logs()
is not True
is False
):
await proxy_logging_obj.db_spend_update_writer.update_database(
token=user_api_key,

22
litellm/proxy/proxy_server.py
View file

@ -4,7 +4,6 @@ import inspect
import io
import os
import random
import secrets
import shutil
import subprocess
import sys
@ -26,7 +25,6 @@ from typing import (
cast,
get_args,
get_origin,
get_type_hints,
)
from litellm._uuid import uuid
@ -34,7 +32,6 @@ from litellm.constants import (
AIOHTTP_CONNECTOR_LIMIT,
AIOHTTP_CONNECTOR_LIMIT_PER_HOST,
AIOHTTP_KEEPALIVE_TIMEOUT,
AIOHTTP_NEEDS_CLEANUP_CLOSED,
AIOHTTP_TTL_DNS_CACHE,
AUDIO_SPEECH_CHUNK_SIZE,
BASE_MCP_ROUTE,
@ -54,7 +51,6 @@ from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
from litellm.types.utils import (
ModelResponse,
ModelResponseStream,
TextCompletionResponse,
TokenCountResponse,
)
from litellm.utils import load_credentials_from_list
@ -1320,22 +1316,12 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False):
def cost_tracking():
global prisma_client
if prisma_client is None:
return
store_proxy_response = ProxyUpdateSpend.should_store_proxy_response_in_spend_logs()
disable_spend = ProxyUpdateSpend.disable_spend_updates()
if store_proxy_response is None and not disable_spend:
verbose_proxy_logger.warning(
"general_settings.spend_logs_store_proxy_response is not set. "
"Current default logs the upstream LLM response; this will change in a future release. "
"Set it to True to store the proxy-mutated response or False to keep current behavior."
proxy_db_logger = _ProxyDBLogger()
litellm.logging_callback_manager.add_litellm_callback(proxy_db_logger)
litellm.logging_callback_manager.add_litellm_async_success_callback(
proxy_db_logger
)
proxy_db_logger = _ProxyDBLogger()
litellm.logging_callback_manager.add_litellm_callback(proxy_db_logger)
litellm.logging_callback_manager.add_litellm_async_success_callback(proxy_db_logger)
async def update_cache( # noqa: PLR0915
token: Optional[str],

105
tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -27,10 +27,16 @@ def mock_proxy_logging_obj(monkeypatch):
def __init__(self):
self.update_database = AsyncMock()
class _ServiceLoggingStub:
def __init__(self):
self.async_service_success_hook = AsyncMock()
self.async_service_failure_hook = AsyncMock()
class _ProxyLoggingStub:
def __init__(self):
self.db_spend_update_writer = _DBWriterStub()
self.slack_alerting_instance = _SlackStub()
self.service_logging_obj = _ServiceLoggingStub()
stub = _ProxyLoggingStub()
monkeypatch.setattr(
@ -47,7 +53,7 @@ def mock_proxy_logging_obj(monkeypatch):
@pytest.mark.asyncio
async def test_async_post_call_failure_hook():
async def test_async_post_call_failure_hook(mock_proxy_logging_obj):
# Setup
logger = _ProxyDBLogger()
@ -74,42 +80,39 @@ async def test_async_post_call_failure_hook():
# Mock exception
original_exception = Exception("Test exception")
# Mock update_database function
with patch(
"litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database",
new_callable=AsyncMock,
) as mock_update_database:
# Call the method
await logger.async_post_call_failure_hook(
request_data=request_data,
original_exception=original_exception,
user_api_key_dict=user_api_key_dict,
)
# Call the method
await logger.async_post_call_failure_hook(
request_data=request_data,
original_exception=original_exception,
user_api_key_dict=user_api_key_dict,
)
# Assertions
mock_update_database.assert_called_once()
update_database = mock_proxy_logging_obj.db_spend_update_writer.update_database
# Check the arguments passed to update_database
call_args = mock_update_database.call_args[1]
print("call_args", json.dumps(call_args, indent=4, default=str))
assert call_args["token"] == "test_api_key"
assert call_args["response_cost"] == 0.0
assert call_args["user_id"] == "test_user_id"
assert call_args["end_user_id"] == "test_end_user_id"
assert call_args["team_id"] == "test_team_id"
assert call_args["org_id"] == "test_org_id"
assert call_args["completion_response"] == original_exception
# Assertions
update_database.assert_awaited_once()
# Check that metadata was properly updated
assert "litellm_params" in call_args["kwargs"]
assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == {
"request_id": "test_request_id"
}
metadata = call_args["kwargs"]["litellm_params"]["metadata"]
assert metadata["user_api_key"] == "test_api_key"
assert metadata["status"] == "failure"
assert "error_information" in metadata
assert metadata["original_key"] == "original_value"
# Check the arguments passed to update_database
call_args = update_database.await_args.kwargs
print("call_args", json.dumps(call_args, indent=4, default=str))
assert call_args["token"] == "test_api_key"
assert call_args["response_cost"] == 0.0
assert call_args["user_id"] == "test_user_id"
assert call_args["end_user_id"] == "test_end_user_id"
assert call_args["team_id"] == "test_team_id"
assert call_args["org_id"] == "test_org_id"
assert call_args["completion_response"] == original_exception
# Check that metadata was properly updated
assert "litellm_params" in call_args["kwargs"]
assert call_args["kwargs"]["litellm_params"]["proxy_server_request"] == {
"request_id": "test_request_id"
}
metadata = call_args["kwargs"]["litellm_params"]["metadata"]
assert metadata["user_api_key"] == "test_api_key"
assert metadata["status"] == "failure"
assert "error_information" in metadata
assert metadata["original_key"] == "original_value"
@pytest.mark.asyncio
@ -161,15 +164,17 @@ async def test_async_post_call_failure_hook_non_llm_route():
async def test_async_log_success_event_writes_db_when_flag_false(monkeypatch, mock_proxy_logging_obj):
logger = _ProxyDBLogger()
metadata = {
"user_api_key": "sk-test",
"user_api_key_user_id": "user-1",
"user_api_key_team_id": "team-1",
"user_api_key_org_id": "org-1",
}
kwargs = {
"model": "gpt-4",
"metadata": {
"user_api_key": "sk-test",
"user_api_key_user_id": "user-1",
"user_api_key_team_id": "team-1",
"user_api_key_org_id": "org-1",
},
"litellm_params": {},
"metadata": metadata,
"litellm_params": {"metadata": metadata.copy()},
"standard_logging_object": StandardLoggingPayload(
id="req-1",
trace_id=None,
@ -359,7 +364,7 @@ async def test_async_log_success_event_with_flag_none(monkeypatch, mock_proxy_lo
end_time=datetime.now(),
)
mock_proxy_logging_obj.db_spend_update_writer.update_database.assert_awaited_once()
mock_proxy_logging_obj.db_spend_update_writer.update_database.assert_not_called()
@pytest.mark.asyncio
@ -370,14 +375,18 @@ async def test_async_post_call_success_hook_writes_db_with_guardrail_info(
class _LoggingObj:
def __init__(self):
metadata = {
"user_api_key": "sk-test",
"user_api_key_user_id": "user-1",
"user_api_key_team_id": "team-1",
"user_api_key_org_id": "org-1",
}
self.model_call_details = {
"metadata": {
"user_api_key": "sk-test",
"user_api_key_user_id": "user-1",
"user_api_key_team_id": "team-1",
"user_api_key_org_id": "org-1",
"metadata": metadata,
"litellm_params": {
"metadata": metadata.copy(),
"litellm_metadata": metadata.copy(),
},
"litellm_params": {},
"standard_logging_object": {
"response_cost": 0.25,
},