scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(proxy): keep guardrail cost in spend on cache hits

Browse source 
The proxy cost callback zeroed response_cost whenever cache_hit was true. That rule dates from Jan 2024 when it was the only place cache hits were priced. The logging layer has priced the LLM share at 0 on a cache hit since Aug 2024, and since guardrail cost joined the standard logging payload the proxy-side zeroing has thrown away a real provider charge: a pre_call guardrail runs before the cache is consulted, so a cached response still cost whatever the guardrail billed. Drop the redundant zeroing so the payload's response_cost, which is already LLM 0 + guardrail cost, reaches spend logs, daily tables and budgets untouched

Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW
This commit is contained in:
ryan-crabbe-berri 2026-09-05 15:32:02 -07:00
parent bf51dea36b
commit acddd21860
2 changed files with 48 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -254,7 +254,7 @@ class _ProxyDBLogger(CustomLogger):
key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None))
end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None)
sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
response_cost = (
response_cost: Final = (
sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None)
)
tags: Final = _get_request_tags_for_cost_tracking(
@ -269,10 +269,6 @@ class _ProxyDBLogger(CustomLogger):
if response_cost is not None:
user_api_key: Final = metadata.get("user_api_key", None)
if kwargs.get("cache_hit", False) is True:
response_cost = 0.0
verbose_proxy_logger.debug("Cache Hit: response_cost %s, for user_id %s", response_cost, user_id)
verbose_proxy_logger.debug(
"user_api_key %s, user_id %s, team_id %s, end_user_id %s",
user_api_key,

47
tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -1783,6 +1783,53 @@ async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata():
)
@pytest.mark.asyncio
async def test_track_cost_callback_keeps_guardrail_cost_on_cache_hit():
"""A cache hit skips the LLM, not the guardrail that screened the prompt, so the
guardrail's provider charge must still reach spend logs and budgets. The payload
already prices the LLM share at 0 on a cache hit, so its response_cost is the
guardrail cost alone and the callback must pass it through untouched."""
logger = _ProxyDBLogger()
kwargs = {
"call_type": "acompletion",
"model": "gpt-4o",
"cache_hit": True,
"response_cost": 0.0,
"litellm_params": {
"metadata": {
"user_api_key": "hashed-key",
"user_api_key_user_id": "user-1",
"user_api_key_team_id": "team-1",
}
},
"standard_logging_object": {
"response_cost": 0.0003,
"request_tags": [],
"metadata": {},
"cost_breakdown": {"guardrail_cost": 0.0003, "total_cost": 0.0003},
},
}
with (
patch("litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock) as mock_increment,
patch("litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock),
patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging,
):
mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock()
mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock()
await logger._PROXY_track_cost_callback(
kwargs=kwargs,
completion_response={"id": "cached-call-1"},
start_time=datetime.now(),
end_time=datetime.now(),
)
update_kwargs = mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs
assert update_kwargs["response_cost"] == pytest.approx(0.0003)
assert mock_increment.call_args.kwargs["response_cost"] == pytest.approx(0.0003)
@pytest.mark.parametrize(
"call_type, expected",
[