fix(agentic loop): keep a recovered retry's cost breakdown on the response it returns

A retried agentic request that recovers no longer adds the failed attempt's first call to its success row. The proxy keeps the follow-up's own row for that request id, so the added spend never showed up in the stored row, and that duplicate row is tracked separately. When every attempt fails, the failure row still bills each attempt's first call

Pricing a first call for the failure row no longer replaces the request's cost breakdown. Before, a recovered retry answered with x-litellm-response-cost-original, -input, and -output from the failed attempt's first call. Now its headers and logged breakdown match a request that succeeds on its first try
This commit is contained in:
mateo-berri 2026-10-30 19:35:12 -07:00
parent b5da3b9bff
commit 4df38d14aa
3 changed files with 40 additions and 63 deletions

View file

@ -50,7 +50,6 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
)
from litellm.cost_calculator import (
BaseTokenUsageProcessor,
RealtimeAPITokenUsageProcessor,
ResponsesWebSocketTokenUsageProcessor,
get_usage_object,
@ -572,7 +571,6 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_REDACTED_RESULT: Final = TypeAdapter(object)
_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[tuple[Usage, float], ...])
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
@ -2234,12 +2232,19 @@ class Logging(LiteLLMLoggingBaseClass):
return 0.0
def uncached_price(self, result: object) -> UncachedPrice:
return self._price_without_recording(result, cache_hit=False, ignore_stamped_cost=True)
def _price_without_recording(
self, result: object, *, cache_hit: bool | None, ignore_stamped_cost: bool
) -> UncachedPrice:
previous_cost_breakdown: Final = self.cost_breakdown
previous_billed_token_rates: Final = self.billed_token_rates
debug_key: Final = "response_cost_failure_debug_information"
previous_failure_debug_info: Final = self.model_call_details.get(debug_key)
try:
cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
cost: Final = self.response_cost_calculator(
result=result, cache_hit=cache_hit, ignore_stamped_cost=ignore_stamped_cost
)
return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
finally:
self.cost_breakdown = previous_cost_breakdown
@ -2670,34 +2675,6 @@ class Logging(LiteLLMLoggingBaseClass):
},
)
def _add_failed_attempts_spend(self, response: object) -> None:
own_usage: Final = get_usage_object(completion_response=response)
own_cost: Final = self.model_call_details.get("response_cost")
if own_usage is None or not isinstance(own_cost, (int, float)):
return
earlier_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(
(
*self.model_call_details.get("failed_attempts_billed_on_success", ()),
*self.model_call_details.pop("failed_attempts_usage", ()),
)
)
if not earlier_attempts:
return
earlier_cost: Final = sum(cost for _, cost in earlier_attempts)
self.model_call_details["failed_attempts_billed_on_success"] = earlier_attempts
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, *(usage for usage, _ in earlier_attempts)]
)
self.model_call_details["response_cost"] = own_cost + earlier_cost
if self.cost_breakdown is not None:
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
total_cost=own_cost + earlier_cost,
additional_costs={
**self.cost_breakdown.get("additional_costs", {}),
"failed_attempts_cost": earlier_cost,
},
)
def claim_stream_failure_report(self) -> bool:
if self._stream_failure_claimed:
return False
@ -2707,7 +2684,10 @@ class Logging(LiteLLMLoggingBaseClass):
def record_earlier_round_for_failure(self, response: object) -> None:
usage: Final = get_usage_object(completion_response=response)
if usage is not None:
self.record_partial_usage_for_failure(usage, self.response_cost_calculator(result=response) or 0.0)
earlier_round_price: Final = self._price_without_recording(
response, cache_hit=None, ignore_stamped_cost=False
)
self.record_partial_usage_for_failure(usage, earlier_round_price.cost or 0.0)
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
@ -2955,7 +2935,6 @@ class Logging(LiteLLMLoggingBaseClass):
elif "response_cost" in hidden_params:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
self._add_failed_attempts_spend(logging_result)
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
):
@ -2965,7 +2944,6 @@ class Logging(LiteLLMLoggingBaseClass):
pass
else:
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
self._add_failed_attempts_spend(logging_result)
if not build_logging_payload:
return None

View file

@ -1918,7 +1918,7 @@ def _router_retrying_once() -> litellm.Router:
@pytest.mark.asyncio
async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success(
async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
_serve_responses(
@ -1945,11 +1945,9 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_th
assert len(recorder.logged) == 1
row: Final = recorder.logged[0]
assert row["id"] == response.id
assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5)
assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19)
assert row["response_cost"] == pytest.approx(
_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY)
)
assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5)
assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens)
assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY))
@pytest.mark.asyncio

View file

@ -3960,34 +3960,35 @@ async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_
assert lifted["response_cost"] == pytest.approx(3 * first_call_cost)
@pytest.mark.asyncio
async def test_retried_agentic_request_that_recovers_bills_the_failed_attempts_first_call_on_success(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
outcome, shared_logging, first_calls, hook = await _route_agentic_request_with_retries(
monkeypatch, respx_mock, ("litellm.RateLimitError", "42")
)
await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10)
async def _successful_agentic_request_cost_report(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...]
) -> tuple[int, Mapping[str, object], Mapping[str, str | None]]:
from litellm.proxy.common_request_processing import cost_headers
outcome, shared_logging, _, hook = await _route_agentic_request_with_retries(monkeypatch, respx_mock, follow_ups)
await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10)
assert isinstance(outcome, litellm.ModelResponse)
success: Final = shared_logging.model_call_details["standard_logging_object"]
follow_up_cost: Final = sum(
litellm.cost_per_token(
model="gpt-4o-mini",
prompt_tokens=outcome.usage.prompt_tokens,
completion_tokens=outcome.usage.completion_tokens,
)
billed: Final = {
field: success[field] for field in ("response_cost", "prompt_tokens", "completion_tokens", "cost_breakdown")
}
return hook.plans_built, billed, cost_headers(success["response_cost"], shared_logging)
@pytest.mark.asyncio
async def test_retried_agentic_request_that_recovers_reports_the_cost_of_a_first_try_success(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
first_try_attempts, first_try_billed, first_try_headers = await _successful_agentic_request_cost_report(
monkeypatch, respx_mock, ("42",)
)
failed_first_call_cost: Final = sum(
litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7)
recovered_attempts, recovered_billed, recovered_headers = await _successful_agentic_request_cost_report(
monkeypatch, respx_mock, ("litellm.RateLimitError", "42")
)
assert first_calls.call_count == 2
assert success["prompt_tokens"] == outcome.usage.prompt_tokens + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"]
assert (
success["completion_tokens"]
== outcome.usage.completion_tokens + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"]
)
assert success["response_cost"] == pytest.approx(follow_up_cost + failed_first_call_cost)
assert (first_try_attempts, recovered_attempts) == (1, 2)
assert recovered_billed == first_try_billed
assert recovered_headers == first_try_headers
def _responses_sse(*events: Mapping[str, object]) -> str: