From 4df38d14aa1971e3ce841b443a41703dc52ea2d5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:35:12 -0700 Subject: [PATCH] fix(agentic loop): keep a recovered retry's cost breakdown on the response it returns A retried agentic request that recovers no longer adds the failed attempt's first call to its success row. The proxy keeps the follow-up's own row for that request id, so the added spend never showed up in the stored row, and that duplicate row is tracked separately. When every attempt fails, the failure row still bills each attempt's first call Pricing a first call for the failure row no longer replaces the request's cost breakdown. Before, a recovered retry answered with x-litellm-response-cost-original, -input, and -output from the failed attempt's first call. Now its headers and logged breakdown match a request that succeeds on its first try --- litellm/litellm_core_utils/litellm_logging.py | 46 +++++------------- .../mcp/test_aresponses_api_with_mcp.py | 10 ++-- tests/unit/test_router/test_router.py | 47 ++++++++++--------- 3 files changed, 40 insertions(+), 63 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3ded219f1ea..a1257340d59 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -50,7 +50,6 @@ from litellm.constants import ( REDACTED_BY_LITELLM, ) from litellm.cost_calculator import ( - BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, get_usage_object, @@ -572,7 +571,6 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata" _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _REDACTED_RESULT: Final = TypeAdapter(object) -_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[tuple[Usage, float], ...]) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -2234,12 +2232,19 @@ class Logging(LiteLLMLoggingBaseClass): return 0.0 def uncached_price(self, result: object) -> UncachedPrice: + return self._price_without_recording(result, cache_hit=False, ignore_stamped_cost=True) + + def _price_without_recording( + self, result: object, *, cache_hit: bool | None, ignore_stamped_cost: bool + ) -> UncachedPrice: previous_cost_breakdown: Final = self.cost_breakdown previous_billed_token_rates: Final = self.billed_token_rates debug_key: Final = "response_cost_failure_debug_information" previous_failure_debug_info: Final = self.model_call_details.get(debug_key) try: - cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True) + cost: Final = self.response_cost_calculator( + result=result, cache_hit=cache_hit, ignore_stamped_cost=ignore_stamped_cost + ) return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown) finally: self.cost_breakdown = previous_cost_breakdown @@ -2670,34 +2675,6 @@ class Logging(LiteLLMLoggingBaseClass): }, ) - def _add_failed_attempts_spend(self, response: object) -> None: - own_usage: Final = get_usage_object(completion_response=response) - own_cost: Final = self.model_call_details.get("response_cost") - if own_usage is None or not isinstance(own_cost, (int, float)): - return - earlier_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python( - ( - *self.model_call_details.get("failed_attempts_billed_on_success", ()), - *self.model_call_details.pop("failed_attempts_usage", ()), - ) - ) - if not earlier_attempts: - return - earlier_cost: Final = sum(cost for _, cost in earlier_attempts) - self.model_call_details["failed_attempts_billed_on_success"] = earlier_attempts - self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( - usage_objects=[own_usage, *(usage for usage, _ in earlier_attempts)] - ) - self.model_call_details["response_cost"] = own_cost + earlier_cost - if self.cost_breakdown is not None: - self.cost_breakdown = self.cost_breakdown | CostBreakdown( - total_cost=own_cost + earlier_cost, - additional_costs={ - **self.cost_breakdown.get("additional_costs", {}), - "failed_attempts_cost": earlier_cost, - }, - ) - def claim_stream_failure_report(self) -> bool: if self._stream_failure_claimed: return False @@ -2707,7 +2684,10 @@ class Logging(LiteLLMLoggingBaseClass): def record_earlier_round_for_failure(self, response: object) -> None: usage: Final = get_usage_object(completion_response=response) if usage is not None: - self.record_partial_usage_for_failure(usage, self.response_cost_calculator(result=response) or 0.0) + earlier_round_price: Final = self._price_without_recording( + response, cache_hit=None, ignore_stamped_cost=False + ) + self.record_partial_usage_for_failure(usage, earlier_round_price.cost or 0.0) def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" @@ -2955,7 +2935,6 @@ class Logging(LiteLLMLoggingBaseClass): elif "response_cost" in hidden_params: self.model_call_details["response_cost"] = hidden_params["response_cost"] self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) - self._add_failed_attempts_spend(logging_result) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and ( existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped ): @@ -2965,7 +2944,6 @@ class Logging(LiteLLMLoggingBaseClass): pass else: self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result) - self._add_failed_attempts_spend(logging_result) if not build_logging_payload: return None diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index d682fef85c3..af9a0b50805 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -1918,7 +1918,7 @@ def _router_retrying_once() -> litellm.Router: @pytest.mark.asyncio -async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success( +async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): _serve_responses( @@ -1945,11 +1945,9 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_th assert len(recorder.logged) == 1 row: Final = recorder.logged[0] assert row["id"] == response.id - assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5) - assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19) - assert row["response_cost"] == pytest.approx( - _round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY) - ) + assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5) + assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens) + assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY)) @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 6651832520c..dc7e8d64889 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -3960,34 +3960,35 @@ async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_ assert lifted["response_cost"] == pytest.approx(3 * first_call_cost) -@pytest.mark.asyncio -async def test_retried_agentic_request_that_recovers_bills_the_failed_attempts_first_call_on_success( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter -) -> None: - outcome, shared_logging, first_calls, hook = await _route_agentic_request_with_retries( - monkeypatch, respx_mock, ("litellm.RateLimitError", "42") - ) - await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) +async def _successful_agentic_request_cost_report( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...] +) -> tuple[int, Mapping[str, object], Mapping[str, str | None]]: + from litellm.proxy.common_request_processing import cost_headers + outcome, shared_logging, _, hook = await _route_agentic_request_with_retries(monkeypatch, respx_mock, follow_ups) + await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) assert isinstance(outcome, litellm.ModelResponse) success: Final = shared_logging.model_call_details["standard_logging_object"] - follow_up_cost: Final = sum( - litellm.cost_per_token( - model="gpt-4o-mini", - prompt_tokens=outcome.usage.prompt_tokens, - completion_tokens=outcome.usage.completion_tokens, - ) + billed: Final = { + field: success[field] for field in ("response_cost", "prompt_tokens", "completion_tokens", "cost_breakdown") + } + return hook.plans_built, billed, cost_headers(success["response_cost"], shared_logging) + + +@pytest.mark.asyncio +async def test_retried_agentic_request_that_recovers_reports_the_cost_of_a_first_try_success( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + first_try_attempts, first_try_billed, first_try_headers = await _successful_agentic_request_cost_report( + monkeypatch, respx_mock, ("42",) ) - failed_first_call_cost: Final = sum( - litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7) + recovered_attempts, recovered_billed, recovered_headers = await _successful_agentic_request_cost_report( + monkeypatch, respx_mock, ("litellm.RateLimitError", "42") ) - assert first_calls.call_count == 2 - assert success["prompt_tokens"] == outcome.usage.prompt_tokens + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] - assert ( - success["completion_tokens"] - == outcome.usage.completion_tokens + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] - ) - assert success["response_cost"] == pytest.approx(follow_up_cost + failed_first_call_cost) + + assert (first_try_attempts, recovered_attempts) == (1, 2) + assert recovered_billed == first_try_billed + assert recovered_headers == first_try_headers def _responses_sse(*events: Mapping[str, object]) -> str: