fix(proxy): report what a failed agentic follow-up's failure row bills on /v1/messages errors

This commit is contained in:
mateo-berri 2026-10-31 03:48:37 -07:00
parent 1531d33a86
commit bc95cab434
2 changed files with 86 additions and 1 deletions

View file

@ -18,6 +18,7 @@ from litellm.anthropic_interface.exceptions import (
AnthropicExceptionMapping,
)
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.tool_fields import first_validation_error
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
from litellm.llms.anthropic.messages.context_management import (
@ -49,6 +50,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
)
from litellm.proxy.common_utils.token_count_log import rejected_token_count
from litellm.proxy.common_utils.token_count_settings import count_source_fields
from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost
from litellm.proxy.litellm_pre_call_utils import (
forwarded_anthropic_oauth_token,
should_auto_drop_params_for_agentic_cli,
@ -284,7 +286,9 @@ async def anthropic_response(
call_id=base_llm_response_processor.litellm_call_id,
model_id=model_id,
version=version,
response_cost=0,
response_cost=(
failure_row_response_cost(base_llm_response_processor.data) if is_agentic_followup_failure(e) else 0
),
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
request_data=data,
timeout=getattr(e, "timeout", None),

View file

@ -2,6 +2,7 @@
Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objects in streaming responses
"""
import datetime
import json
import logging
import unittest
@ -248,6 +249,86 @@ class TestBlockedResponseUsage:
assert response["content"] == [{"type": "text", "text": _BLOCK_TEXT}]
class TestAnthropicResponseReportsWhatAFailedAgenticFollowUpBills:
async def _error_response_cost(self, raise_the_follow_up_error: bool) -> float:
import litellm
import litellm.proxy.anthropic_endpoints.endpoints as ep
import litellm.proxy.proxy_server as proxy_server
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
from litellm.proxy.proxy_server import ProxyConfig
from litellm.proxy.utils import ProxyLogging
from litellm.types.utils import ModelResponse, Usage
proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging.alert_types = []
follow_up_error: Final = litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o")
async def failing_follow_up() -> ModelResponse:
raise follow_up_error
async def fail_after_the_first_call(self: ProxyBaseLLMRequestProcessing, **kwargs: object) -> None:
_, logging_obj = await self.common_processing_pre_call_logic(
request=kwargs["request"],
general_settings={},
user_api_key_dict=kwargs["user_api_key_dict"],
proxy_logging_obj=proxy_logging,
proxy_config=MagicMock(spec=ProxyConfig),
route_type="anthropic_messages",
)
logging_obj.model_call_details.update(first_api_call_start_time=datetime.datetime.now())
with pytest.raises(litellm.RateLimitError):
await bill_first_round_on_failure(
failing_follow_up(),
logging_obj=logging_obj,
response=ModelResponse(
model="gpt-4o", usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672)
),
stream=False,
)
raise (
follow_up_error
if raise_the_follow_up_error
else litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o")
)
request: Final = MagicMock(spec=Request)
request.headers = {}
with (
patch.object(
ep,
"read_request_body",
new=AsyncMock(
return_value={
"model": "gpt-4o",
"max_tokens": 64,
"messages": [{"role": "user", "content": "what is 6*7?"}],
}
),
),
patch.object(ep.ProxyBaseLLMRequestProcessing, "base_process_llm_request", new=fail_after_the_first_call),
patch.object(proxy_server, "proxy_logging_obj", proxy_logging),
):
response: Final = await ep.anthropic_response(
fastapi_response=Response(),
request=request,
user_api_key_dict=UserAPIKeyAuth(request_route="/v1/messages"),
)
assert response.status_code == 429
return float(response.headers["x-litellm-response-cost"])
@pytest.mark.asyncio
async def test_the_error_response_reports_the_served_first_call_its_failure_row_bills(self) -> None:
import litellm
price: Final = litellm.model_cost["gpt-4o"]
assert await self._error_response_cost(raise_the_follow_up_error=True) == pytest.approx(
559 * price["input_cost_per_token"] + 113 * price["output_cost_per_token"]
)
assert await self._error_response_cost(raise_the_follow_up_error=False) == 0
class TestProxyExceptionAnthropicEnvelope:
@pytest.mark.asyncio
async def test_anthropic_response_maps_proxy_exception_to_anthropic_envelope(self):