mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fix(interactions): stop the unpollable-create path from firing a false cost-tracking alert
The tightened gate correctly stopped deferring the reservation release for InteractionsAPIResponses the scheduler will not poll (terminal status, or in_progress without an id), but the response then fell through into the generic 'Cost tracking failed' raise and the failed_tracking_alert path. A create returning failed, cancelled, requires_action, incomplete, budget_exceeded, or an id-less in_progress without usage therefore released its reservation as intended and, in the same breath, alerted operators for a legitimate no-usage response, both creating noise and masking real cost-tracking failures. The two gates are now nested under a single 'unbilled interaction response' outer check, so any InteractionsAPIResponse with no usage takes either the defer path (pollable, polling on) or the release-and-return path, and none of them fall through to the generic failure raise. The two regression tests also now assert failed_tracking_alert is not called, closing the observation gap the report flagged.
This commit is contained in:
parent
8b566a7f0a
commit
0ab8ef60bf
2 changed files with 20 additions and 5 deletions
|
|
@ -319,8 +319,10 @@ class _ProxyDBLogger(CustomLogger):
|
|||
elif budget_reservation is not None:
|
||||
await _release_budget_reservation(budget_reservation=budget_reservation)
|
||||
else:
|
||||
if _is_unbilled_in_progress_interaction(completion_response):
|
||||
if BACKGROUND_INTERACTION_COST_POLLING_ENABLED:
|
||||
if _is_unbilled_interaction_response(completion_response):
|
||||
if BACKGROUND_INTERACTION_COST_POLLING_ENABLED and _is_unbilled_in_progress_interaction(
|
||||
completion_response
|
||||
):
|
||||
verbose_proxy_logger.debug(
|
||||
"Cost tracking deferred for in-progress background interaction; "
|
||||
"the budget reservation stays open until the poll task logs the final usage"
|
||||
|
|
@ -328,8 +330,8 @@ class _ProxyDBLogger(CustomLogger):
|
|||
return
|
||||
await _release_budget_reservation(budget_reservation=budget_reservation)
|
||||
verbose_proxy_logger.debug(
|
||||
"Background interaction cost polling is disabled; released the budget "
|
||||
"reservation for an in-progress interaction that will not be billed"
|
||||
"Released the budget reservation for an interaction create with no usage "
|
||||
"that no poll task will settle"
|
||||
)
|
||||
return
|
||||
await _release_budget_reservation(budget_reservation=budget_reservation)
|
||||
|
|
@ -477,6 +479,12 @@ def _write_spend_metadata_to_kwargs(kwargs: dict, metadata: dict) -> None:
|
|||
bucket[key] = value
|
||||
|
||||
|
||||
def _is_unbilled_interaction_response(completion_response: object) -> bool:
|
||||
from litellm.types.interactions import InteractionsAPIResponse
|
||||
|
||||
return isinstance(completion_response, InteractionsAPIResponse) and completion_response.usage is None
|
||||
|
||||
|
||||
def _is_unbilled_in_progress_interaction(completion_response: object) -> bool:
|
||||
from litellm.interactions.background_cost_polling import is_pollable_background_interaction
|
||||
from litellm.types.interactions import InteractionsAPIResponse
|
||||
|
|
|
|||
|
|
@ -871,6 +871,10 @@ async def test_track_cost_callback_releases_reservation_for_unpollable_interacti
|
|||
callback must release it there and then, or the pre-call estimate stays
|
||||
added to the key, user, team and org spend counters and starts refusing
|
||||
traffic against budget that was never actually spent.
|
||||
|
||||
A no-usage terminal create is also not a cost-tracking failure, so the
|
||||
callback must not fire ``failed_tracking_alert``: doing so would flood
|
||||
operators with false alerts and mask real cost-tracking failures.
|
||||
"""
|
||||
from litellm.types.interactions import InteractionsAPIResponse
|
||||
|
||||
|
|
@ -897,6 +901,7 @@ async def test_track_cost_callback_releases_reservation_for_unpollable_interacti
|
|||
)
|
||||
|
||||
assert reservation["finalized"] is True
|
||||
mock_proxy_logging.failed_tracking_alert.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -904,7 +909,8 @@ async def test_track_cost_callback_releases_reservation_for_interaction_without_
|
|||
"""
|
||||
The scheduler also refuses a response with no id, since it has nothing to
|
||||
poll for, so the callback must not defer to a poll task that will never
|
||||
exist.
|
||||
exist, and it must not fire ``failed_tracking_alert`` for what is a
|
||||
legitimate no-usage response rather than a cost-tracking failure.
|
||||
"""
|
||||
from litellm.types.interactions import InteractionsAPIResponse
|
||||
|
||||
|
|
@ -931,6 +937,7 @@ async def test_track_cost_callback_releases_reservation_for_interaction_without_
|
|||
)
|
||||
|
||||
assert reservation["finalized"] is True
|
||||
mock_proxy_logging.failed_tracking_alert.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue