mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(agentic loop): keep a recovered retry's cost breakdown on the response it returns
A retried agentic request that recovers no longer adds the failed attempt's first call to its success row. The proxy keeps the follow-up's own row for that request id, so the added spend never showed up in the stored row, and that duplicate row is tracked separately. When every attempt fails, the failure row still bills each attempt's first call Pricing a first call for the failure row no longer replaces the request's cost breakdown. Before, a recovered retry answered with x-litellm-response-cost-original, -input, and -output from the failed attempt's first call. Now its headers and logged breakdown match a request that succeeds on its first try
This commit is contained in:
parent
b5da3b9bff
commit
4df38d14aa
3 changed files with 40 additions and 63 deletions
|
|
@ -50,7 +50,6 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
)
|
||||
from litellm.cost_calculator import (
|
||||
BaseTokenUsageProcessor,
|
||||
RealtimeAPITokenUsageProcessor,
|
||||
ResponsesWebSocketTokenUsageProcessor,
|
||||
get_usage_object,
|
||||
|
|
@ -572,7 +571,6 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"
|
|||
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
|
||||
_REDACTED_RESULT: Final = TypeAdapter(object)
|
||||
_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[tuple[Usage, float], ...])
|
||||
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
|
||||
|
||||
|
||||
|
|
@ -2234,12 +2232,19 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return 0.0
|
||||
|
||||
def uncached_price(self, result: object) -> UncachedPrice:
|
||||
return self._price_without_recording(result, cache_hit=False, ignore_stamped_cost=True)
|
||||
|
||||
def _price_without_recording(
|
||||
self, result: object, *, cache_hit: bool | None, ignore_stamped_cost: bool
|
||||
) -> UncachedPrice:
|
||||
previous_cost_breakdown: Final = self.cost_breakdown
|
||||
previous_billed_token_rates: Final = self.billed_token_rates
|
||||
debug_key: Final = "response_cost_failure_debug_information"
|
||||
previous_failure_debug_info: Final = self.model_call_details.get(debug_key)
|
||||
try:
|
||||
cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True)
|
||||
cost: Final = self.response_cost_calculator(
|
||||
result=result, cache_hit=cache_hit, ignore_stamped_cost=ignore_stamped_cost
|
||||
)
|
||||
return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown)
|
||||
finally:
|
||||
self.cost_breakdown = previous_cost_breakdown
|
||||
|
|
@ -2670,34 +2675,6 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
},
|
||||
)
|
||||
|
||||
def _add_failed_attempts_spend(self, response: object) -> None:
|
||||
own_usage: Final = get_usage_object(completion_response=response)
|
||||
own_cost: Final = self.model_call_details.get("response_cost")
|
||||
if own_usage is None or not isinstance(own_cost, (int, float)):
|
||||
return
|
||||
earlier_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(
|
||||
(
|
||||
*self.model_call_details.get("failed_attempts_billed_on_success", ()),
|
||||
*self.model_call_details.pop("failed_attempts_usage", ()),
|
||||
)
|
||||
)
|
||||
if not earlier_attempts:
|
||||
return
|
||||
earlier_cost: Final = sum(cost for _, cost in earlier_attempts)
|
||||
self.model_call_details["failed_attempts_billed_on_success"] = earlier_attempts
|
||||
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
|
||||
usage_objects=[own_usage, *(usage for usage, _ in earlier_attempts)]
|
||||
)
|
||||
self.model_call_details["response_cost"] = own_cost + earlier_cost
|
||||
if self.cost_breakdown is not None:
|
||||
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
|
||||
total_cost=own_cost + earlier_cost,
|
||||
additional_costs={
|
||||
**self.cost_breakdown.get("additional_costs", {}),
|
||||
"failed_attempts_cost": earlier_cost,
|
||||
},
|
||||
)
|
||||
|
||||
def claim_stream_failure_report(self) -> bool:
|
||||
if self._stream_failure_claimed:
|
||||
return False
|
||||
|
|
@ -2707,7 +2684,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
def record_earlier_round_for_failure(self, response: object) -> None:
|
||||
usage: Final = get_usage_object(completion_response=response)
|
||||
if usage is not None:
|
||||
self.record_partial_usage_for_failure(usage, self.response_cost_calculator(result=response) or 0.0)
|
||||
earlier_round_price: Final = self._price_without_recording(
|
||||
response, cache_hit=None, ignore_stamped_cost=False
|
||||
)
|
||||
self.record_partial_usage_for_failure(usage, earlier_round_price.cost or 0.0)
|
||||
|
||||
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
|
||||
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
|
||||
|
|
@ -2955,7 +2935,6 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
elif "response_cost" in hidden_params:
|
||||
self.model_call_details["response_cost"] = hidden_params["response_cost"]
|
||||
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
|
||||
self._add_failed_attempts_spend(logging_result)
|
||||
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
|
||||
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
|
||||
):
|
||||
|
|
@ -2965,7 +2944,6 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
pass
|
||||
else:
|
||||
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
|
||||
self._add_failed_attempts_spend(logging_result)
|
||||
|
||||
if not build_logging_payload:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1918,7 +1918,7 @@ def _router_retrying_once() -> litellm.Router:
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success(
|
||||
async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
_serve_responses(
|
||||
|
|
@ -1945,11 +1945,9 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_th
|
|||
assert len(recorder.logged) == 1
|
||||
row: Final = recorder.logged[0]
|
||||
assert row["id"] == response.id
|
||||
assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5)
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19)
|
||||
assert row["response_cost"] == pytest.approx(
|
||||
_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY)
|
||||
)
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5)
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens)
|
||||
assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -3960,34 +3960,35 @@ async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_
|
|||
assert lifted["response_cost"] == pytest.approx(3 * first_call_cost)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retried_agentic_request_that_recovers_bills_the_failed_attempts_first_call_on_success(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
outcome, shared_logging, first_calls, hook = await _route_agentic_request_with_retries(
|
||||
monkeypatch, respx_mock, ("litellm.RateLimitError", "42")
|
||||
)
|
||||
await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10)
|
||||
async def _successful_agentic_request_cost_report(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...]
|
||||
) -> tuple[int, Mapping[str, object], Mapping[str, str | None]]:
|
||||
from litellm.proxy.common_request_processing import cost_headers
|
||||
|
||||
outcome, shared_logging, _, hook = await _route_agentic_request_with_retries(monkeypatch, respx_mock, follow_ups)
|
||||
await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10)
|
||||
assert isinstance(outcome, litellm.ModelResponse)
|
||||
success: Final = shared_logging.model_call_details["standard_logging_object"]
|
||||
follow_up_cost: Final = sum(
|
||||
litellm.cost_per_token(
|
||||
model="gpt-4o-mini",
|
||||
prompt_tokens=outcome.usage.prompt_tokens,
|
||||
completion_tokens=outcome.usage.completion_tokens,
|
||||
)
|
||||
billed: Final = {
|
||||
field: success[field] for field in ("response_cost", "prompt_tokens", "completion_tokens", "cost_breakdown")
|
||||
}
|
||||
return hook.plans_built, billed, cost_headers(success["response_cost"], shared_logging)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retried_agentic_request_that_recovers_reports_the_cost_of_a_first_try_success(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
first_try_attempts, first_try_billed, first_try_headers = await _successful_agentic_request_cost_report(
|
||||
monkeypatch, respx_mock, ("42",)
|
||||
)
|
||||
failed_first_call_cost: Final = sum(
|
||||
litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7)
|
||||
recovered_attempts, recovered_billed, recovered_headers = await _successful_agentic_request_cost_report(
|
||||
monkeypatch, respx_mock, ("litellm.RateLimitError", "42")
|
||||
)
|
||||
assert first_calls.call_count == 2
|
||||
assert success["prompt_tokens"] == outcome.usage.prompt_tokens + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"]
|
||||
assert (
|
||||
success["completion_tokens"]
|
||||
== outcome.usage.completion_tokens + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"]
|
||||
)
|
||||
assert success["response_cost"] == pytest.approx(follow_up_cost + failed_first_call_cost)
|
||||
|
||||
assert (first_try_attempts, recovered_attempts) == (1, 2)
|
||||
assert recovered_billed == first_try_billed
|
||||
assert recovered_headers == first_try_headers
|
||||
|
||||
|
||||
def _responses_sse(*events: Mapping[str, object]) -> str:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue