fix(agentic): stop retrying a request whose agentic follow-up failed and bill every round it served

An error raised by an agentic follow-up after the first call was served now carries a typed mark, so the
router's retry loops, the mid-stream retry, and litellm's num_retries wrapper raise it as is instead of
rerunning the whole request. A failure of the first call itself still retries, and fallbacks still run.

The failure row bills every round the loop served before the failure, at any depth, and a later attempt
that succeeds bills what the failed attempts paid on its success row, with the cost breakdown and the
response cost headers adding up. The error response's x-litellm-response-cost reports the same amount
its failure row bills. Failure rows outside agentic hooks keep the prompt-token estimate they had before.

A refused tool call now matches by id among the refused ids, or by name among refused calls that carry
no id, so a name-only refusal next to an id refusal is no longer kept.
This commit is contained in:
mateo-berri 2026-10-31 00:32:12 -07:00
parent 4df38d14aa
commit 7fd14e5beb
16 changed files with 711 additions and 243 deletions

View file

@ -0,0 +1,27 @@
import weakref
from typing import Final, TypeAlias
from litellm.types.utils import Usage
ServedRound: TypeAlias = tuple[Usage, float]
_followup_failures: Final[weakref.WeakKeyDictionary[BaseException, tuple[ServedRound, ...]]] = (
weakref.WeakKeyDictionary()
)
def _can_be_marked(error: BaseException) -> bool:
return type(error).__weakrefoffset__ != 0
def mark_agentic_followup_failure(error: BaseException, served_rounds: tuple[ServedRound, ...]) -> None:
if _can_be_marked(error):
_followup_failures[error] = served_rounds
def is_agentic_followup_failure(error: BaseException) -> bool:
return _can_be_marked(error) and error in _followup_failures
def rounds_served_before(error: BaseException) -> tuple[ServedRound, ...]:
return _followup_failures.get(error, ()) if _can_be_marked(error) else ()

View file

@ -40,19 +40,19 @@ async def every_claimed_tool_call(
return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))})
def _call_ids(call: object) -> frozenset[str]:
return _strings((message_field(call, "id"), message_field(call, "call_id")))
def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
calls: Final = _claimed_calls(tool_calls)
ids: Final = _strings(
chain.from_iterable((message_field(call, "id"), message_field(call, "call_id")) for call in calls)
)
names: Final = _strings(message_field(call, "name") for call in calls)
ids: Final = frozenset(chain.from_iterable(_call_ids(call) for call in calls))
names: Final = _strings(message_field(call, "name") for call in calls if not _call_ids(call))
return ids, names
def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
if isinstance(call_id, str) and refused_ids:
return call_id in refused_ids
return name in refused_names
return (isinstance(call_id, str) and call_id in refused_ids) or name in refused_names
def is_refused_item(

View file

@ -10,6 +10,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure, rounds_served_before
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn
from litellm.litellm_core_utils.agentic_loop_settings import (
@ -54,9 +55,14 @@ async def bill_first_round_on_failure(
) -> _DispatchResult:
try:
return await dispatch
except Exception:
if not stream and isinstance(logging_obj, LiteLLMLoggingObject):
logging_obj.record_earlier_round_for_failure(response)
except Exception as error:
if stream or not isinstance(logging_obj, LiteLLMLoggingObject):
mark_agentic_followup_failure(error, rounds_served_before(error))
raise
this_round: Final = logging_obj.served_round(response)
served_rounds: Final = (*rounds_served_before(error), *(() if this_round is None else (this_round,)))
mark_agentic_followup_failure(error, served_rounds)
logging_obj.record_served_rounds_for_failure(served_rounds)
raise

View file

@ -50,6 +50,7 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
)
from litellm.cost_calculator import (
BaseTokenUsageProcessor,
RealtimeAPITokenUsageProcessor,
ResponsesWebSocketTokenUsageProcessor,
get_usage_object,
@ -70,6 +71,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.langtrace import langtrace_trace_endpoint
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.agentic_followup_failure import ServedRound
from litellm.litellm_core_utils.classifier_logging import (
classifier_audit_fields,
classifier_input_snapshot,
@ -571,6 +573,7 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
_REDACTED_RESULT: Final = TypeAdapter(object)
_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[ServedRound, ...])
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
@ -2636,35 +2639,43 @@ class Logging(LiteLLMLoggingBaseClass):
own usage while the request's failure log still bills every attempt."""
usage: Final = self._model_call_details_view.get("combined_usage_object")
response_cost: Final = self._model_call_details_view.get("response_cost")
billed_by_continuation: Final = self.model_call_details.pop("stash_billed_by_continuation", False) is True
self.model_call_details.pop("combined_usage_object", None)
self.model_call_details.pop("response_cost", None)
if isinstance(usage, Usage):
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
if not isinstance(usage, Usage):
return
attempt: Final = (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0)
self.model_call_details["current_attempt_api_call_start_time"] = None
self.model_call_details["failed_attempts_usage"] = (
*self.model_call_details.get("failed_attempts_usage", ()),
attempt,
)
if not billed_by_continuation:
self.model_call_details["failed_attempts_owed_on_success"] = (
*self.model_call_details.get("failed_attempts_owed_on_success", ()),
attempt,
)
def mark_stash_billed_by_continuation(self) -> None:
self.model_call_details["stash_billed_by_continuation"] = True
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
self.model_call_details["continued_partial_usage"] = usage
self.model_call_details["continued_partial_cost"] = response_cost
self.model_call_details["continued_response_id"] = response_id
def _add_continued_partial_spend(self, response: object) -> None:
def _add_continued_partial_spend(self, response: object) -> Usage | None:
partial_usage: Final = self.model_call_details.get("continued_partial_usage")
partial_cost: Final = self.model_call_details.get("continued_partial_cost")
own_usage: Final = getattr(response, "usage", None)
own_cost: Final = self.model_call_details.get("response_cost")
if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
return
from litellm.cost_calculator import BaseTokenUsageProcessor
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, partial_usage]
)
return own_usage if isinstance(own_usage, Usage) else None
combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[own_usage, partial_usage])
self.model_call_details["combined_usage_object"] = combined_usage
if not isinstance(own_cost, (int, float)):
return
return combined_usage
self.model_call_details["response_cost"] = own_cost + partial_cost
if self.cost_breakdown is not None:
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
@ -2674,6 +2685,35 @@ class Logging(LiteLLMLoggingBaseClass):
"mid_stream_partial_cost": partial_cost,
},
)
return combined_usage
def _add_failed_attempts_spend(self, own_usage: Usage | None) -> None:
own_cost: Final = self.model_call_details.get("response_cost")
if own_usage is None or not isinstance(own_cost, (int, float)):
return
failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(
(
*self.model_call_details.get("failed_attempts_billed_on_success", ()),
*self.model_call_details.pop("failed_attempts_owed_on_success", ()),
)
)
if not failed_attempts:
return
failed_attempts_cost: Final = sum(cost for _, cost in failed_attempts)
self.model_call_details.pop("failed_attempts_usage", None)
self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)]
)
self.model_call_details["response_cost"] = own_cost + failed_attempts_cost
if self.cost_breakdown is not None:
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
total_cost=own_cost + failed_attempts_cost,
additional_costs={
**self.cost_breakdown.get("additional_costs", {}),
"failed_attempts_cost": failed_attempts_cost,
},
)
def claim_stream_failure_report(self) -> bool:
if self._stream_failure_claimed:
@ -2681,13 +2721,22 @@ class Logging(LiteLLMLoggingBaseClass):
self._stream_failure_claimed = True
return True
def record_earlier_round_for_failure(self, response: object) -> None:
def served_round(self, response: object) -> ServedRound | None:
usage: Final = get_usage_object(completion_response=response)
if usage is not None:
earlier_round_price: Final = self._price_without_recording(
response, cache_hit=None, ignore_stamped_cost=False
)
self.record_partial_usage_for_failure(usage, earlier_round_price.cost or 0.0)
if usage is None:
return None
price: Final = self._price_without_recording(response, cache_hit=None, ignore_stamped_cost=False)
return usage, price.cost or 0.0
def record_served_rounds_for_failure(self, served_rounds: tuple[ServedRound, ...]) -> None:
if not served_rounds:
return
self.record_partial_usage_for_failure(
served_rounds[0][0]
if len(served_rounds) == 1
else BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[usage for usage, _ in served_rounds]),
sum(cost for _, cost in served_rounds),
)
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
@ -2935,6 +2984,7 @@ class Logging(LiteLLMLoggingBaseClass):
elif "response_cost" in hidden_params:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
):
@ -2944,6 +2994,7 @@ class Logging(LiteLLMLoggingBaseClass):
pass
else:
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
if not build_logging_payload:
return None
@ -3449,7 +3500,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = self.response_cost_calculator(
result=complete_streaming_response
)
self._add_continued_partial_spend(complete_streaming_response)
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
logged_streaming_response: Final = self._with_continued_response_id(
self._with_client_facing_stream_model(complete_streaming_response)
@ -4041,7 +4092,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.model,
)
self.model_call_details["response_cost"] = None
self._add_continued_partial_spend(complete_streaming_response)
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)

View file

@ -53,6 +53,7 @@ from litellm.constants import (
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.integrations.otel.runtime import phase_event
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.bug_report import (
allowlisted,
bug_report_notice,
@ -129,6 +130,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.proxy.route_llm_request import (
endpoint_unsupported_by_model,
@ -4342,6 +4344,7 @@ class ProxyBaseLLMRequestProcessing:
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
log_llm_api_exception(e, self.litellm_call_id)
billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj")
failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e)
# Allow callbacks to transform the error response
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
@ -4378,7 +4381,9 @@ class ProxyBaseLLMRequestProcessing:
model_id=model_id,
version=version,
response_cost=(
_litellm_logging_obj.finalized_cost.amount
failure_row_response_cost(self.data)
if failed_in_agentic_follow_up
else _litellm_logging_obj.finalized_cost.amount
if isinstance(_litellm_logging_obj, LiteLLMLoggingObj)
and _litellm_logging_obj.finalized_cost is not None
else 0

View file

@ -218,6 +218,32 @@ def _guardrail_info_lifted_from_litellm_metadata(
return {"standard_logging_guardrail_information": guardrail_info}
def _failure_row_guardrail_information(request_data: Mapping[str, object]) -> object:
metadata: Final = _str_keyed_fields(request_data.get("metadata"))
lifted: Final = _guardrail_info_lifted_from_litellm_metadata(
_str_keyed_fields(request_data.get("litellm_metadata")),
guardrail_info_already_set="standard_logging_guardrail_information" in metadata,
)
return {**metadata, **lifted}.get("standard_logging_guardrail_information")
def failure_row_response_cost(request_data: Mapping[str, object]) -> float:
stashed_cost: Final = request_data.get("response_cost")
recovered_stream_cost: Final = (
max(float(stashed_cost), 0.0)
if isinstance(request_data.get("combined_usage_object"), litellm.Usage)
and isinstance(stashed_cost, (int, float))
else 0.0
)
finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object"))
finalized_amount: Final = finalized_failure.get("response_cost")
return (
float(finalized_amount)
if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float))
else recovered_stream_cost + guardrail_information_cost(_failure_row_guardrail_information(request_data))
)
def _redaction_opt_in_lifted_from_litellm_metadata(
litellm_metadata: ObjectMapping, *, caller_set_headers: bool
) -> ObjectMapping:
@ -543,19 +569,7 @@ class _ProxyDBLogger(CustomLogger):
# recovered cost onto request_data (the usage rides along in
# ``combined_usage_object`` for the token columns), so attribute the
# real partial spend to this failure row instead of zero.
recovered_stream_cost: Final = (
max(float(request_data.get("response_cost") or 0.0), 0.0)
if isinstance(request_data.get("combined_usage_object"), litellm.Usage)
else 0.0
)
finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object"))
finalized_amount: Final = finalized_failure.get("response_cost")
recovered_response_cost: Final = (
float(finalized_amount)
if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float))
else recovered_stream_cost
+ guardrail_information_cost(existing_metadata.get("standard_logging_guardrail_information"))
)
recovered_response_cost: Final = failure_row_response_cost(request_data)
failure_snapshot_with_late_guardrails: Final = _failure_snapshot_with_guardrails_recorded_after_it(
request_data.get("standard_logging_object"),

View file

@ -1359,16 +1359,16 @@ def _failure_usage_to_lift(
base_model: str | None = None,
dispatched_call_type: str | None = None,
) -> tuple[object, object] | None:
failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ()))
current_attempt: Final = _current_failure_usage_to_lift(
model_call_details,
request_body,
dispatched and (provider_billed or not failed_attempts),
dispatched,
provider_billed,
deployment_id,
base_model,
dispatched_call_type=dispatched_call_type,
)
failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ()))
if not failed_attempts:
return current_attempt
current_usage, current_cost = current_attempt if current_attempt is not None else (None, None)

View file

@ -106,6 +106,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.routing import routing_decision_attributes
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.batch_utils import (
batch_create_routing,
@ -3902,6 +3903,8 @@ class Router(DeprecatedRouterMembers):
if continuation_output is not None
else None
)
if cut_partial is not None:
model_response.logging_obj.mark_stash_billed_by_continuation()
if continue_after_content:
emitted_tokens: Final = int(
getattr(complete_response_object_usage, "completion_tokens", 0) or 0
@ -7123,6 +7126,8 @@ class Router(DeprecatedRouterMembers):
last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see
for attempt in range(attempted_retries_for_request(initial_kwargs), budget):
trigger = _mid_stream_retry_trigger(last_error)
if is_agentic_followup_failure(trigger):
return _StreamRetriesExhausted(last_error)
if not policy_applies and not self._stream_should_retry(
trigger, healthy_deployments, all_deployments, initial_kwargs
):
@ -9339,7 +9344,7 @@ class Router(DeprecatedRouterMembers):
response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None)
return response
except Exception as e:
if is_guardrail_intervention(e):
if is_guardrail_intervention(e) or is_agentic_followup_failure(e):
raise
current_attempt = None
original_exception = e
@ -9446,6 +9451,8 @@ class Router(DeprecatedRouterMembers):
return response
except Exception as e:
if is_agentic_followup_failure(e):
raise
# Always track the latest error so we raise the most
# recent exception instead of the first one.
original_exception = e

View file

@ -83,6 +83,7 @@ from litellm.constants import (
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
)
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
from litellm.litellm_core_utils.core_helpers import (
bind_budget_reservation_to_callbacks,
normalize_drop_params,
@ -2235,7 +2236,7 @@ def client(original_function):
is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
if (
num_retries and not is_completion_litellm_router_call
num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
if (
isinstance(e, openai.APIError)
@ -2269,7 +2270,7 @@ def client(original_function):
is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
if (
num_retries and not is_responses_litellm_router_call
num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
if (
isinstance(e, openai.APIError)
@ -2554,7 +2555,10 @@ def client(original_function):
is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True)
if (
num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError)
num_retries
and not is_acompletion_litellm_router_call
and not isinstance(e, ImportError)
and not is_agentic_followup_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
try:
litellm.num_retries = None # set retries to None to prevent infinite loops
@ -2585,7 +2589,10 @@ def client(original_function):
is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True)
if (
num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError)
num_retries
and not is_aresponses_litellm_router_call
and not isinstance(e, ImportError)
and not is_agentic_followup_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
try:
litellm.num_retries = None # set retries to None to prevent infinite loops

View file

@ -22,6 +22,7 @@ they stay out of the body even without it.
import datetime
import json
from collections.abc import Mapping, Sequence
from typing import Any, Dict, Final, List, Optional, Tuple
from unittest.mock import AsyncMock, MagicMock, patch
@ -264,8 +265,6 @@ class _GateOnlyLogger(CustomLogger):
class _RecordingGateLogger(_GateOnlyLogger):
"""Also keeps every follow-up response the loop hands to the post hook."""
def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None:
super().__init__(plan=plan, tool_calls=tool_calls)
self.followups: tuple[ModelResponse, ...] = ()
@ -622,6 +621,35 @@ async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clien
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
@pytest.mark.asyncio
async def test_refused_rerun_drops_a_call_claimed_only_by_name_next_to_one_claimed_by_id(restore_callbacks):
response: Final = ModelResponse(
choices=[
Choices(
finish_reason="tool_calls",
message=Message(
role="assistant",
content=None,
tool_calls=[
_tool_call("call_abc", "litellm_code_execution"),
_tool_call("call_search", "litellm_web_search"),
_tool_call("call_client", "lookup"),
],
),
)
]
)
result: Final = await _run_refused_rerun(
response,
_REFUSING_RAILS[0],
gate_tool_calls={"tool_calls": [{"id": "call_abc"}]},
other_hooks_tool_calls=({"tool_calls": [{"name": "litellm_web_search"}]},),
)
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response, gate_tool_calls",
@ -716,6 +744,118 @@ async def test_acompletion_bills_the_first_call_on_the_failure_log_when_the_foll
)
def _tool_call_turn(call_id: str, usage: Usage) -> ModelResponse:
return ModelResponse(
model="gpt-4o-mini",
choices=[
Choices(
finish_reason="tool_calls",
message=Message(role="assistant", content=None, tool_calls=[_tool_call(call_id, "litellm_web_search")]),
)
],
usage=usage,
)
class _TwoRoundLogger(CustomLogger):
def __init__(self, middle_round: ModelResponse) -> None:
super().__init__()
self._follow_ups: Final = (middle_round, "litellm.RateLimitError")
self.logging_objs: tuple[object, ...] = ()
async def async_should_run_agentic_loop(
self,
response: object,
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
assert isinstance(response, ModelResponse)
calls: Final = response.choices[0].message.tool_calls or []
return bool(calls), {"tool_calls": [{"id": call.id} for call in calls]}
async def async_build_agentic_loop_plan(
self,
tools: Mapping[str, object],
model: str,
messages: Sequence[Mapping[str, object]],
response: object,
anthropic_messages_provider_config: object,
anthropic_messages_optional_request_params: Mapping[str, object],
logging_obj: object,
stream: bool,
kwargs: Mapping[str, object],
) -> AgenticLoopPlan:
follow_up: Final = self._follow_ups[len(self.logging_objs)]
self.logging_objs = (*self.logging_objs, logging_obj)
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": follow_up}),
)
@pytest.mark.asyncio
async def test_acompletion_bills_every_served_round_on_the_failure_log_when_a_later_follow_up_fails(
restore_callbacks, monkeypatch: pytest.MonkeyPatch
):
first_usage: Final = Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)
middle_usage: Final = Usage(prompt_tokens=11, completion_tokens=4, total_tokens=15)
logger: Final = _TwoRoundLogger(middle_round=_tool_call_turn("call_next", middle_usage))
monkeypatch.setattr(litellm, "callbacks", [logger])
mock_client: Final = MagicMock()
mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_turn("call_abc", first_usage)))
with pytest.raises(litellm.RateLimitError):
await litellm.acompletion(
model="openai/gpt-4o-mini",
messages=[{"role": "user", "content": "what is 6*7?"}],
api_key="sk-test",
client=mock_client,
num_retries=0,
)
first_logging_obj, _ = logger.logging_objs
assert isinstance(first_logging_obj, Logging)
price: Final = litellm.model_cost["gpt-4o-mini"]
billed_usage: Final = first_logging_obj.model_call_details["combined_usage_object"]
assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (5 + 11, 3 + 4)
assert first_logging_obj.model_call_details["response_cost"] == pytest.approx(
(5 + 11) * price["input_cost_per_token"] + (3 + 4) * price["output_cost_per_token"]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
async def test_a_follow_up_failing_with_a_builtin_error_reaches_the_caller_unchanged(stream: bool):
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
sandbox_error: Final = RuntimeError("sandbox crashed")
async def failing_follow_up() -> ModelResponse:
raise sandbox_error
with pytest.raises(RuntimeError) as raised:
await bill_first_round_on_failure(
failing_follow_up(),
logging_obj=Logging(
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
stream=stream,
call_type="acompletion",
start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
litellm_call_id="call-builtin-error",
function_id="fn-builtin-error",
),
response=_tool_call_turn("call_abc", Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)),
stream=stream,
)
assert raised.value is sandbox_error
class _RunOnlyLoopLogger(CustomLogger):
def __init__(self, result: ModelResponse) -> None:
super().__init__()

View file

@ -5,7 +5,7 @@ import json
import logging
import threading
import time
from collections.abc import Awaitable, Callable, Mapping
from collections.abc import Awaitable, Callable, Mapping, Sequence
from types import MappingProxyType
from typing import Final
from unittest.mock import AsyncMock, Mock, patch
@ -5369,12 +5369,12 @@ class _PlannedChatLoop(CustomLogger):
self,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict,
) -> tuple[bool, dict]:
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
claimed: Final = [
{"id": call.id, "name": call.function.name}
for call in _tool_calls_of(response)
@ -5384,14 +5384,14 @@ class _PlannedChatLoop(CustomLogger):
async def async_build_chat_completion_agentic_loop_plan(
self,
tools: dict,
tools: Mapping[str, object],
model: str,
messages: list[dict],
messages: Sequence[Mapping[str, object]],
response: object,
optional_params: dict,
optional_params: Mapping[str, object],
logging_obj: object,
stream: bool,
kwargs: dict,
kwargs: Mapping[str, object],
) -> AgenticLoopPlan:
self.logging_objs.append(logging_obj)
return self._plan
@ -5406,24 +5406,24 @@ class _RunOnlyChatLoop(CustomLogger):
self,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict,
) -> tuple[bool, dict]:
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
return True, {"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]}
async def async_run_chat_completion_agentic_loop(
self,
tools: dict,
tools: Mapping[str, object],
model: str,
messages: list[dict],
messages: Sequence[Mapping[str, object]],
response: object,
optional_params: dict,
optional_params: Mapping[str, object],
logging_obj: object,
stream: bool,
kwargs: dict,
kwargs: Mapping[str, object],
) -> object:
return self._result
@ -5599,12 +5599,12 @@ class _SearchAgainLoop(CustomLogger):
self,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict,
) -> tuple[bool, dict]:
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
searches: Final = [
{"id": call.id, "name": call.function.name}
for call in response.choices[0].message.tool_calls or []
@ -5614,14 +5614,14 @@ class _SearchAgainLoop(CustomLogger):
async def async_build_chat_completion_agentic_loop_plan(
self,
tools: dict,
tools: Mapping[str, object],
model: str,
messages: list[dict],
messages: Sequence[Mapping[str, object]],
response: object,
optional_params: dict,
optional_params: Mapping[str, object],
logging_obj: object,
stream: bool,
kwargs: dict,
kwargs: Mapping[str, object],
) -> AgenticLoopPlan:
return AgenticLoopPlan(
run_agentic_loop=True,

View file

@ -3,7 +3,7 @@ import gzip
import importlib
import json
import os
from collections.abc import Awaitable, Callable, Mapping
from collections.abc import Awaitable, Callable, Mapping, Sequence
from types import ModuleType
from typing import Final, Literal, Protocol
from unittest.mock import MagicMock, Mock, patch
@ -236,12 +236,12 @@ class _SearchingChatGate(CustomLogger):
self,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict,
) -> tuple[bool, dict]:
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
return True, {"tool_calls": [{"id": "call-1"}]}
@ -253,14 +253,14 @@ class _FailingFollowUpChatLoop(_SearchingChatGate):
async def async_run_chat_completion_agentic_loop(
self,
tools: dict,
tools: Mapping[str, object],
model: str,
messages: list[dict],
messages: Sequence[Mapping[str, object]],
response: object,
optional_params: dict,
optional_params: Mapping[str, object],
logging_obj: object,
stream: bool,
kwargs: dict,
kwargs: Mapping[str, object],
) -> object:
self.runs += 1
self.logging_objs.append(logging_obj)
@ -272,26 +272,26 @@ class _RaisingAgenticGate(CustomLogger):
self,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict,
) -> tuple[bool, dict]:
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
raise ValueError("gate bug")
class _AnsweringChatLoop(_SearchingChatGate):
async def async_run_chat_completion_agentic_loop(
self,
tools: dict,
tools: Mapping[str, object],
model: str,
messages: list[dict],
messages: Sequence[Mapping[str, object]],
response: object,
optional_params: dict,
optional_params: Mapping[str, object],
logging_obj: object,
stream: bool,
kwargs: dict,
kwargs: Mapping[str, object],
) -> object:
self.runs += 1
return litellm.ModelResponse(
@ -490,12 +490,12 @@ class _SearchAgainLoop(CustomLogger):
self,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict,
) -> tuple[bool, dict]:
kwargs: Mapping[str, object],
) -> tuple[bool, dict[str, object]]:
searches: Final = [
{"id": call.id, "name": call.function.name}
for call in response.choices[0].message.tool_calls or []
@ -505,14 +505,14 @@ class _SearchAgainLoop(CustomLogger):
async def async_build_chat_completion_agentic_loop_plan(
self,
tools: dict,
tools: Mapping[str, object],
model: str,
messages: list[dict],
messages: Sequence[Mapping[str, object]],
response: object,
optional_params: dict,
optional_params: Mapping[str, object],
logging_obj: object,
stream: bool,
kwargs: dict,
kwargs: Mapping[str, object],
) -> AgenticLoopPlan:
return AgenticLoopPlan(
run_agentic_loop=True,

View file

@ -4292,6 +4292,62 @@ class TestHandleLLMApiExceptionPricesWhatTheServerDispatched:
assert resent >= dispatched_after_trim
class TestHandleLLMApiExceptionReportsWhatAFailedAgenticFollowUpBills:
async def _error_response_cost(self, raise_the_follow_up_error: bool) -> float:
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
from litellm.types.utils import ModelResponse, Usage
processing_obj: Final = ProxyBaseLLMRequestProcessing(
data={"model": "gpt-4o", "messages": [{"role": "user", "content": "what is 6*7?"}]}
)
caller: Final = ProxyUserAPIKeyAuth(request_route="/v1/chat/completions")
mock_request: Final = MagicMock(spec=Request)
mock_request.headers = {}
proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache())
proxy_logging.alert_types = []
_, logging_obj = await processing_obj.common_processing_pre_call_logic(
request=mock_request,
general_settings={},
user_api_key_dict=caller,
proxy_logging_obj=proxy_logging,
proxy_config=MagicMock(spec=ProxyConfig),
route_type="acompletion",
)
logging_obj.model_call_details.update(first_api_call_start_time=datetime.datetime.now())
follow_up_error: Final = litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o")
async def failing_follow_up() -> ModelResponse:
raise follow_up_error
with pytest.raises(litellm.RateLimitError):
await bill_first_round_on_failure(
failing_follow_up(),
logging_obj=logging_obj,
response=ModelResponse(
model="gpt-4o", usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672)
),
stream=False,
)
with pytest.raises(ProxyException) as raised:
await processing_obj.handle_llm_api_exception(
e=follow_up_error
if raise_the_follow_up_error
else litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o"),
user_api_key_dict=caller,
proxy_logging_obj=proxy_logging,
)
return float(raised.value.headers["x-litellm-response-cost"])
@pytest.mark.asyncio
async def test_the_error_response_reports_the_served_first_call_its_failure_row_bills(self) -> None:
price: Final = litellm.model_cost["gpt-4o"]
assert await self._error_response_cost(raise_the_follow_up_error=True) == pytest.approx(
559 * price["input_cost_per_token"] + 113 * price["output_cost_per_token"]
)
assert await self._error_response_cost(raise_the_follow_up_error=False) == 0
@pytest.mark.asyncio
class TestHandleLLMApiExceptionDictDetail:
"""

View file

@ -3561,13 +3561,15 @@ def test_an_earlier_timed_out_responses_attempt_counts_its_input_from_the_newest
[None, 1700000001.0],
ids=["retry-not-dispatched", "retry-dispatched"],
)
def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attempts(
def test_a_retry_failure_bills_input_only_for_dispatched_attempts(
current_attempt_api_call_start_time: float | None,
) -> None:
from litellm.proxy.utils import _failure_fields_to_lift
from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift
from litellm.types.utils import Usage
prior_usage: Final = Usage(prompt_tokens=4, completion_tokens=2, total_tokens=6)
estimated_retry_usage: Final = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None)
assert estimated_retry_usage is not None
lifted: Final = _failure_fields_to_lift(
{
@ -3586,43 +3588,63 @@ def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attemp
usage: Final = lifted["combined_usage_object"]
assert isinstance(usage, Usage)
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (4, 2, 6)
expected_retry_tokens: Final = (
estimated_retry_usage.prompt_tokens if current_attempt_api_call_start_time is not None else 0
)
assert usage.prompt_tokens == prior_usage.prompt_tokens + expected_retry_tokens
assert usage.completion_tokens == prior_usage.completion_tokens
assert usage.total_tokens == prior_usage.total_tokens + expected_retry_tokens
assert lifted["response_cost"] == 0.25
def test_a_responses_failure_after_a_retried_follow_up_bills_only_the_first_calls_usage() -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
@pytest.mark.asyncio
async def test_an_agentic_follow_up_failure_row_bills_each_served_first_call_without_an_estimate() -> None:
import litellm
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
from litellm.proxy.utils import _failure_fields_to_lift
from litellm.types.utils import ModelResponse, Usage
responses_input: Final = [{"role": "user", "content": "Search the web for the latest LiteLLM release"}]
logging_obj: Final = Logging(
model=FAILURE_USAGE_MODEL,
messages=responses_input,
messages=ONE_USER_MESSAGE,
stream=False,
call_type="aresponses",
call_type="acompletion",
start_time=real_datetime.datetime.now(),
litellm_call_id="call-responses-retry",
function_id="fn-responses-retry",
litellm_call_id="call-agentic-follow-up",
function_id="fn-agentic-follow-up",
)
logging_obj.update_environment_variables(
model=FAILURE_USAGE_MODEL, user="", optional_params={}, litellm_params={"custom_llm_provider": "openai"}
)
logging_obj.pre_call(input=responses_input, api_key="sk-test")
first_call: Final = ModelResponse(usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672))
logging_obj.record_earlier_round_for_failure(first_call)
first_call_cost: Final = logging_obj.model_call_details["response_cost"]
logging_obj.carry_failed_attempt_usage()
logging_obj.pre_call(input=responses_input, api_key="sk-test")
lifted: Final = _failure_fields_to_lift(
{"litellm_logging_obj": logging_obj}, provider_billed=False, dispatched_call_type="aresponses"
async def failing_follow_up() -> ModelResponse:
raise litellm.RateLimitError(message="rate limited", llm_provider="openai", model=FAILURE_USAGE_MODEL)
async def serve_first_call_then_fail_its_follow_up(usage: Usage) -> float:
logging_obj.pre_call(input=ONE_USER_MESSAGE, api_key="sk-test")
with pytest.raises(litellm.RateLimitError):
await bill_first_round_on_failure(
failing_follow_up(),
logging_obj=logging_obj,
response=ModelResponse(model=FAILURE_USAGE_MODEL, usage=usage),
stream=False,
)
return logging_obj.model_call_details["response_cost"]
primary_cost: Final = await serve_first_call_then_fail_its_follow_up(
Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672)
)
logging_obj.carry_failed_attempt_usage()
backup_cost: Final = await serve_first_call_then_fail_its_follow_up(
Usage(prompt_tokens=31, completion_tokens=5, total_tokens=36)
)
lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logging_obj})
usage: Final = lifted["combined_usage_object"]
assert (usage.prompt_tokens, usage.completion_tokens) == (559, 113)
assert first_call_cost > 0
assert lifted["response_cost"] == pytest.approx(first_call_cost)
assert (usage.prompt_tokens, usage.completion_tokens) == (559 + 31, 113 + 5)
assert primary_cost > 0 and backup_cost > 0
assert lifted["response_cost"] == pytest.approx(primary_cost + backup_cost)
@pytest.mark.parametrize(

View file

@ -26,7 +26,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.xai.common_utils import xai_reported_cost_in_usd
from litellm.proxy.mcp_server import mcp_server_manager
from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk
from litellm.proxy.utils import _failure_fields_to_lift
from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator
from litellm.types.llms.openai import (
@ -540,13 +540,9 @@ async def test_mcp_allowed_tools_filtering():
]
# First filter by allowed tools
filtered_tools_with_duplicates = (
LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
mcp_tools=mock_mcp_tools_with_duplicates,
mcp_tools_with_litellm_proxy=cast(
List[ToolParam], mcp_tool_config_with_duplicates
),
)
filtered_tools_with_duplicates = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
mcp_tools=mock_mcp_tools_with_duplicates,
mcp_tools_with_litellm_proxy=cast(List[ToolParam], mcp_tool_config_with_duplicates),
)
# Then deduplicate the filtered tools
@ -605,11 +601,9 @@ async def test_mcp_allowed_tools_filtering():
},
]
filtered_tools_multiple = (
LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
mcp_tools=mock_mcp_tools_from_server,
mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs),
)
filtered_tools_multiple = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
mcp_tools=mock_mcp_tools_from_server,
mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs),
)
# Should return union of all allowed tools (3 unique tools)
@ -1918,7 +1912,7 @@ def _router_retrying_once() -> litellm.Router:
@pytest.mark.asyncio
async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds(
async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
):
_serve_responses(
@ -1945,9 +1939,11 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_ea
assert len(recorder.logged) == 1
row: Final = recorder.logged[0]
assert row["id"] == response.id
assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5)
assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens)
assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY))
assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5)
assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19)
assert row["response_cost"] == pytest.approx(
_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY)
)
@pytest.mark.asyncio
@ -1982,7 +1978,22 @@ async def test_non_stream_auto_execute_failing_every_retry_bills_each_paid_first
failure_row_fields: Final = _failure_fields_to_lift(MappingProxyType({"litellm_logging_obj": logging_obj}))
billed_usage: Final = failure_row_fields["combined_usage_object"]
assert isinstance(billed_usage, Usage)
assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == billed_tokens
estimated_failure_usage: Final = (
_estimate_dispatched_failure_usage(
model="gpt-5.6",
request_input=logging_obj.model_call_details.get("messages"),
system_input=None,
)
if not retried_first_rounds
else None
)
expected_prompt_tokens: Final = billed_tokens[0] + (
estimated_failure_usage.prompt_tokens if estimated_failure_usage is not None else 0
)
assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (
expected_prompt_tokens,
billed_tokens[1],
)
assert failure_row_fields["response_cost"] == pytest.approx(
_round_cost(_ROUND_ONE_BODY) + sum(_round_cost(body) for body in retried_first_rounds)
)

View file

@ -3652,14 +3652,28 @@ class _CutSSE(httpx.AsyncByteStream):
raise httpx.RemoteProtocolError("peer closed connection without sending complete message body")
_BACKUP_SSE: Final = "".join(
f"data: {json.dumps(chunk)}\n\n"
for chunk in (
{"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest",
"choices": [{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}]},
{"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest",
"choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}},
_BACKUP_SSE: Final = (
"".join(
f"data: {json.dumps(chunk)}\n\n"
for chunk in (
{
"id": "b",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "mistral-large-latest",
"choices": [
{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}
],
},
{
"id": "b",
"object": "chat.completion.chunk",
"created": 1700000000,
"model": "mistral-large-latest",
"choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96},
},
)
)
) + "data: [DONE]\n\n"
@ -3907,88 +3921,165 @@ class _ScriptedAgenticFollowUps(CustomLogger):
self.shared_call_succeeded.set()
async def _route_agentic_request_with_retries(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...]
_AGENTIC_BACKUP_USAGE: Final = {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23}
_AGENTIC_BACKUP_BODY: Final = {
"id": "chatcmpl-backup",
"object": "chat.completion",
"created": 1700000000,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "42"}}],
"usage": _AGENTIC_BACKUP_USAGE,
}
_AGENTIC_RATE_LIMITED: Final = {
"error": {"message": "Rate limit reached", "type": "requests", "code": "rate_limit_exceeded"}
}
async def _route_agentic_request(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
follow_ups: tuple[str, ...],
model: str = "m",
first_call_replies: tuple[httpx.Response, ...] = (),
fallbacks: tuple[str, ...] = (),
) -> tuple[object, LiteLLMLogging, respx.Route, _ScriptedAgenticFollowUps]:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
hook: Final = _ScriptedAgenticFollowUps(follow_ups)
monkeypatch.setattr(litellm, "callbacks", [hook])
first_calls: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY)
first_calls: Final = (
respx_mock.post("https://api.openai.com/v1/chat/completions").mock(side_effect=list(first_call_replies))
if first_call_replies
else respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY)
)
)
respx_mock.post("https://backup.test/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_AGENTIC_BACKUP_BODY)
)
router: Final = Router(
model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}],
model_list=[
{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}},
{
"model_name": "backup",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "sk-test",
"api_base": "https://backup.test/v1",
},
},
],
num_retries=2,
retry_after=0,
fallbacks=[{"m": list(fallbacks)}] if fallbacks else [],
)
messages: Final = [{"role": "user", "content": "what is 6*7?"}]
shared_logging, _ = litellm.utils.function_setup(
original_function="acompletion",
rules_obj=litellm.utils.Rules(),
start_time=datetime.now(),
model="m",
model=model,
messages=messages,
litellm_call_id="shared-agentic-call",
)
try:
outcome: object = await router.acompletion(
model="m", messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging
model=model, messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging
)
except litellm.RateLimitError as error:
outcome = error
return outcome, shared_logging, first_calls, hook
def _gpt_4o_mini_cost(usage: Mapping[str, int]) -> float:
return sum(
litellm.cost_per_token(
model="gpt-4o-mini", prompt_tokens=usage["prompt_tokens"], completion_tokens=usage["completion_tokens"]
)
)
@pytest.mark.asyncio
async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_first_call(
async def test_agentic_request_whose_follow_up_fails_is_not_retried_and_bills_its_one_first_call(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
from litellm.proxy.utils import _failure_fields_to_lift
outcome, shared_logging, first_calls, _ = await _route_agentic_request_with_retries(
outcome, shared_logging, first_calls, _ = await _route_agentic_request(
monkeypatch, respx_mock, ("litellm.RateLimitError",)
)
lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": shared_logging})
first_call_cost: Final = sum(
litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7)
)
assert isinstance(outcome, litellm.RateLimitError)
assert first_calls.call_count == 3
assert lifted["combined_usage_object"].prompt_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"]
assert lifted["combined_usage_object"].completion_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["completion_tokens"]
assert lifted["response_cost"] == pytest.approx(3 * first_call_cost)
async def _successful_agentic_request_cost_report(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...]
) -> tuple[int, Mapping[str, object], Mapping[str, str | None]]:
from litellm.proxy.common_request_processing import cost_headers
outcome, shared_logging, _, hook = await _route_agentic_request_with_retries(monkeypatch, respx_mock, follow_ups)
await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10)
assert isinstance(outcome, litellm.ModelResponse)
success: Final = shared_logging.model_call_details["standard_logging_object"]
billed: Final = {
field: success[field] for field in ("response_cost", "prompt_tokens", "completion_tokens", "cost_breakdown")
}
return hook.plans_built, billed, cost_headers(success["response_cost"], shared_logging)
assert outcome.status_code == 429
assert first_calls.call_count == 1
assert lifted["combined_usage_object"].prompt_tokens == _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"]
assert lifted["combined_usage_object"].completion_tokens == _AGENTIC_FIRST_CALL_USAGE["completion_tokens"]
assert lifted["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE))
@pytest.mark.asyncio
async def test_retried_agentic_request_that_recovers_reports_the_cost_of_a_first_try_success(
async def test_agentic_request_whose_first_call_fails_is_still_retried(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
first_try_attempts, first_try_billed, first_try_headers = await _successful_agentic_request_cost_report(
monkeypatch, respx_mock, ("42",)
)
recovered_attempts, recovered_billed, recovered_headers = await _successful_agentic_request_cost_report(
monkeypatch, respx_mock, ("litellm.RateLimitError", "42")
outcome, _, first_calls, hook = await _route_agentic_request(
monkeypatch,
respx_mock,
("42",),
first_call_replies=(
httpx.Response(429, json=_AGENTIC_RATE_LIMITED),
httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY),
),
)
assert (first_try_attempts, recovered_attempts) == (1, 2)
assert recovered_billed == first_try_billed
assert recovered_headers == first_try_headers
assert isinstance(outcome, litellm.ModelResponse)
assert first_calls.call_count == 2
assert hook.plans_built == 1
async def _successful_agentic_request_billing(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
follow_ups: tuple[str, ...],
model: str,
fallbacks: tuple[str, ...] = (),
) -> tuple[Mapping[str, object], Mapping[str, str | None]]:
from litellm.proxy.common_request_processing import cost_headers
outcome, shared_logging, _, hook = await _route_agentic_request(
monkeypatch, respx_mock, follow_ups, model=model, fallbacks=fallbacks
)
await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10)
assert isinstance(outcome, litellm.ModelResponse)
success: Final = shared_logging.model_call_details["standard_logging_object"]
return success, cost_headers(success["response_cost"], shared_logging)
@pytest.mark.asyncio
async def test_fallback_after_a_failed_agentic_follow_up_bills_the_primarys_first_call_on_the_success_row(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
) -> None:
first_try, first_try_headers = await _successful_agentic_request_billing(
monkeypatch, respx_mock, ("42",), model="backup"
)
recovered, recovered_headers = await _successful_agentic_request_billing(
monkeypatch, respx_mock, ("litellm.RateLimitError",), model="m", fallbacks=("backup",)
)
first_call_cost: Final = _gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE)
assert first_try["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_BACKUP_USAGE))
assert "failed_attempts_cost" not in first_try["cost_breakdown"].get("additional_costs", {})
assert recovered["response_cost"] == pytest.approx(first_try["response_cost"] + first_call_cost)
assert recovered["prompt_tokens"] == first_try["prompt_tokens"] + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"]
assert (
recovered["completion_tokens"]
== first_try["completion_tokens"] + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"]
)
assert recovered["cost_breakdown"]["total_cost"] == pytest.approx(recovered["response_cost"])
assert recovered["cost_breakdown"]["additional_costs"]["failed_attempts_cost"] == pytest.approx(first_call_cost)
assert float(recovered_headers["x-litellm-response-cost"]) == pytest.approx(recovered["response_cost"])
assert {key: value for key, value in recovered_headers.items() if key != "x-litellm-response-cost"} == {
key: value for key, value in first_try_headers.items() if key != "x-litellm-response-cost"
}
def _responses_sse(*events: Mapping[str, object]) -> str:
@ -13817,7 +13908,11 @@ def test_get_search_tool_support_reports_each_routable_deployment_of_the_group()
@pytest.mark.parametrize("provider", ["chatgpt", "github_copilot"])
@pytest.mark.parametrize("explicit_provider", [False, True])
async def test_search_tool_catalog_lookup_needs_no_credentials(
provider: str, explicit_provider: bool, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
provider: str,
explicit_provider: bool,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "chatgpt"))
monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path / "copilot"))
@ -17485,13 +17580,16 @@ def test_model_group_info_fast_mode_fails_closed_without_explicit_boolean(
def test_model_group_info_fast_mode_requires_every_deployment(
local_model_cost_map: None, other_model: str, expected: bool, reverse: bool
) -> None:
models: Final = (other_model, "anthropic/claude-opus-5") if reverse else (
"anthropic/claude-opus-5", other_model
models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ("anthropic/claude-opus-5", other_model)
router: Final = Router(
model_list=[
{
"model_name": "fast-group",
"litellm_params": {"model": model, "api_key": "fake-key"},
}
for model in models
]
)
router: Final = Router(model_list=[{
"model_name": "fast-group",
"litellm_params": {"model": model, "api_key": "fake-key"},
} for model in models])
result: Final = router.get_model_group_info("fast-group")
@ -17849,10 +17947,7 @@ class _AnthropicMessagesFallbackByteStream:
def _anthropic_messages_overloaded_error_chunk() -> bytes:
return (
b"event: error\n"
b'data: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n'
)
return b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n'
def _anthropic_messages_invalid_request_error_chunk() -> bytes:
@ -24600,16 +24695,25 @@ class TestAutoRouterTraceProvenance:
monkeypatch.setattr(litellm, "_async_success_callback", [logger, CallbackComplete()])
classifier: Final = ObservedClassifier()
router: Final = Router(model_list=[
{**self._marker("blue"), "litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"classifier_type": "custom", "classifier_plugin": classifier,
"tiers": {"SIMPLE": "answer"}, "session_affinity": False, "deployment_affinity": False,
router: Final = Router(
model_list=[
{
**self._marker("blue"),
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"classifier_type": "custom",
"classifier_plugin": classifier,
"tiers": {"SIMPLE": "answer"},
"session_affinity": False,
"deployment_affinity": False,
},
},
},
}},
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
], plugins=[classifier])
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
],
plugins=[classifier],
)
stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"}
request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale, "user_api_key_team_id": "team"}}
with logger.tracer.start_as_current_span("request") as root:
@ -24694,20 +24798,22 @@ class TestAutoRouterTraceProvenance:
self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]"
) -> None:
logger, exporter = _tracing
router: Final = Router(model_list=[
self._marker("blue"),
{
"model_name": "traced-router",
"model_info": {"id": "definition-semantic"},
"litellm_params": {
"model": "auto_router/semantic-router",
"tags": ["blue"],
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
"auto_router_default_model": "answer",
"auto_router_embedding_model": "embed",
router: Final = Router(
model_list=[
self._marker("blue"),
{
"model_name": "traced-router",
"model_info": {"id": "definition-semantic"},
"litellm_params": {
"model": "auto_router/semantic-router",
"tags": ["blue"],
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
"auto_router_default_model": "answer",
"auto_router_embedding_model": "embed",
},
},
},
])
]
)
with logger.start_phase_span("route traced-router"):
response: Final = await router.async_pre_routing_hook(
model="traced-router", request_kwargs={"metadata": {"tags": ["blue"]}}
@ -24727,26 +24833,42 @@ class TestAutoRouterTraceProvenance:
) -> None:
logger, exporter = _tracing
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://embedding.example/v1/embeddings").respond(200, json={
"object": "list",
"model": "text-embedding-3-small",
"data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
"usage": {"prompt_tokens": 1, "total_tokens": 1},
})
router: Final = Router(model_list=[
{
"model_name": "internal-semantic-router",
"model_info": {"id": "semantic-definition", "team_id": "trace-team", "team_public_model_name": "public-router"},
"litellm_params": {
"model": "auto_router/semantic-router",
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
"auto_router_default_model": "answer",
"auto_router_embedding_model": "embed",
},
respx_mock.post("https://embedding.example/v1/embeddings").respond(
200,
json={
"object": "list",
"model": "text-embedding-3-small",
"data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
"usage": {"prompt_tokens": 1, "total_tokens": 1},
},
{"model_name": "embed", "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", "api_key": "test"}},
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
])
)
router: Final = Router(
model_list=[
{
"model_name": "internal-semantic-router",
"model_info": {
"id": "semantic-definition",
"team_id": "trace-team",
"team_public_model_name": "public-router",
},
"litellm_params": {
"model": "auto_router/semantic-router",
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
"auto_router_default_model": "answer",
"auto_router_embedding_model": "embed",
},
},
{
"model_name": "embed",
"litellm_params": {
"model": "openai/text-embedding-3-small",
"api_base": "https://embedding.example/v1",
"api_key": "test",
},
},
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
]
)
request: Final = {"metadata": {"user_api_key_team_id": "trace-team"}}
with logger.tracer.start_as_current_span("request"):
await router.async_get_available_deployment(