From 7fd14e5beb43aaef97bd192b308a4b972d8cc8a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:32:12 -0700 Subject: [PATCH] fix(agentic): stop retrying a request whose agentic follow-up failed and bill every round it served An error raised by an agentic follow-up after the first call was served now carries a typed mark, so the router's retry loops, the mid-stream retry, and litellm's num_retries wrapper raise it as is instead of rerunning the whole request. A failure of the first call itself still retries, and fallbacks still run. The failure row bills every round the loop served before the failure, at any depth, and a later attempt that succeeds bills what the failed attempts paid on its success row, with the cost breakdown and the response cost headers adding up. The error response's x-litellm-response-cost reports the same amount its failure row bills. Failure rows outside agentic hooks keep the prompt-token estimate they had before. A refused tool call now matches by id among the refused ids, or by name among refused calls that carry no id, so a name-only refusal next to an id refusal is no longer kept. --- .../agentic_followup_failure.py | 27 ++ .../agentic_loop_refusal.py | 14 +- .../chat_completion_agentic_loop.py | 12 +- litellm/litellm_core_utils/litellm_logging.py | 93 +++-- litellm/proxy/common_request_processing.py | 7 +- .../proxy/hooks/proxy_track_cost_callback.py | 40 ++- litellm/proxy/utils.py | 4 +- litellm/router.py | 9 +- litellm/utils.py | 15 +- .../test_chat_completion_agentic_loop.py | 144 +++++++- .../custom_httpx/test_llm_http_handler.py | 50 +-- tests/unit/llms/openai/test_openai.py | 50 +-- .../proxy/test_common_request_processing.py | 56 +++ ..._utils_model_creation_and_error_logging.py | 64 ++-- .../mcp/test_aresponses_api_with_mcp.py | 47 ++- tests/unit/test_router/test_router.py | 322 ++++++++++++------ 16 files changed, 711 insertions(+), 243 deletions(-) create mode 100644 litellm/litellm_core_utils/agentic_followup_failure.py diff --git a/litellm/litellm_core_utils/agentic_followup_failure.py b/litellm/litellm_core_utils/agentic_followup_failure.py new file mode 100644 index 00000000000..1847d0f0cd7 --- /dev/null +++ b/litellm/litellm_core_utils/agentic_followup_failure.py @@ -0,0 +1,27 @@ +import weakref +from typing import Final, TypeAlias + +from litellm.types.utils import Usage + +ServedRound: TypeAlias = tuple[Usage, float] + +_followup_failures: Final[weakref.WeakKeyDictionary[BaseException, tuple[ServedRound, ...]]] = ( + weakref.WeakKeyDictionary() +) + + +def _can_be_marked(error: BaseException) -> bool: + return type(error).__weakrefoffset__ != 0 + + +def mark_agentic_followup_failure(error: BaseException, served_rounds: tuple[ServedRound, ...]) -> None: + if _can_be_marked(error): + _followup_failures[error] = served_rounds + + +def is_agentic_followup_failure(error: BaseException) -> bool: + return _can_be_marked(error) and error in _followup_failures + + +def rounds_served_before(error: BaseException) -> tuple[ServedRound, ...]: + return _followup_failures.get(error, ()) if _can_be_marked(error) else () diff --git a/litellm/litellm_core_utils/agentic_loop_refusal.py b/litellm/litellm_core_utils/agentic_loop_refusal.py index abd31e856f6..f3ee15dffdd 100644 --- a/litellm/litellm_core_utils/agentic_loop_refusal.py +++ b/litellm/litellm_core_utils/agentic_loop_refusal.py @@ -40,19 +40,19 @@ async def every_claimed_tool_call( return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))}) +def _call_ids(call: object) -> frozenset[str]: + return _strings((message_field(call, "id"), message_field(call, "call_id"))) + + def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: calls: Final = _claimed_calls(tool_calls) - ids: Final = _strings( - chain.from_iterable((message_field(call, "id"), message_field(call, "call_id")) for call in calls) - ) - names: Final = _strings(message_field(call, "name") for call in calls) + ids: Final = frozenset(chain.from_iterable(_call_ids(call) for call in calls)) + names: Final = _strings(message_field(call, "name") for call in calls if not _call_ids(call)) return ids, names def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: - if isinstance(call_id, str) and refused_ids: - return call_id in refused_ids - return name in refused_names + return (isinstance(call_id, str) and call_id in refused_ids) or name in refused_names def is_refused_item( diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 3021d417b9e..7de27f104f6 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -10,6 +10,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure, rounds_served_before from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn from litellm.litellm_core_utils.agentic_loop_settings import ( @@ -54,9 +55,14 @@ async def bill_first_round_on_failure( ) -> _DispatchResult: try: return await dispatch - except Exception: - if not stream and isinstance(logging_obj, LiteLLMLoggingObject): - logging_obj.record_earlier_round_for_failure(response) + except Exception as error: + if stream or not isinstance(logging_obj, LiteLLMLoggingObject): + mark_agentic_followup_failure(error, rounds_served_before(error)) + raise + this_round: Final = logging_obj.served_round(response) + served_rounds: Final = (*rounds_served_before(error), *(() if this_round is None else (this_round,))) + mark_agentic_followup_failure(error, served_rounds) + logging_obj.record_served_rounds_for_failure(served_rounds) raise diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a1257340d59..edb7856fc55 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -50,6 +50,7 @@ from litellm.constants import ( REDACTED_BY_LITELLM, ) from litellm.cost_calculator import ( + BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, get_usage_object, @@ -70,6 +71,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.langtrace import langtrace_trace_endpoint from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger +from litellm.litellm_core_utils.agentic_followup_failure import ServedRound from litellm.litellm_core_utils.classifier_logging import ( classifier_audit_fields, classifier_input_snapshot, @@ -571,6 +573,7 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata" _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _REDACTED_RESULT: Final = TypeAdapter(object) +_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[ServedRound, ...]) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -2636,35 +2639,43 @@ class Logging(LiteLLMLoggingBaseClass): own usage while the request's failure log still bills every attempt.""" usage: Final = self._model_call_details_view.get("combined_usage_object") response_cost: Final = self._model_call_details_view.get("response_cost") + billed_by_continuation: Final = self.model_call_details.pop("stash_billed_by_continuation", False) is True self.model_call_details.pop("combined_usage_object", None) self.model_call_details.pop("response_cost", None) - if isinstance(usage, Usage): - self.model_call_details["current_attempt_api_call_start_time"] = None - self.model_call_details["failed_attempts_usage"] = ( - *self.model_call_details.get("failed_attempts_usage", ()), - (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0), + if not isinstance(usage, Usage): + return + attempt: Final = (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0) + self.model_call_details["current_attempt_api_call_start_time"] = None + self.model_call_details["failed_attempts_usage"] = ( + *self.model_call_details.get("failed_attempts_usage", ()), + attempt, + ) + if not billed_by_continuation: + self.model_call_details["failed_attempts_owed_on_success"] = ( + *self.model_call_details.get("failed_attempts_owed_on_success", ()), + attempt, ) + def mark_stash_billed_by_continuation(self) -> None: + self.model_call_details["stash_billed_by_continuation"] = True + def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None: """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw.""" self.model_call_details["continued_partial_usage"] = usage self.model_call_details["continued_partial_cost"] = response_cost self.model_call_details["continued_response_id"] = response_id - def _add_continued_partial_spend(self, response: object) -> None: + def _add_continued_partial_spend(self, response: object) -> Usage | None: partial_usage: Final = self.model_call_details.get("continued_partial_usage") partial_cost: Final = self.model_call_details.get("continued_partial_cost") own_usage: Final = getattr(response, "usage", None) own_cost: Final = self.model_call_details.get("response_cost") if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)): - return - from litellm.cost_calculator import BaseTokenUsageProcessor - - self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( - usage_objects=[own_usage, partial_usage] - ) + return own_usage if isinstance(own_usage, Usage) else None + combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[own_usage, partial_usage]) + self.model_call_details["combined_usage_object"] = combined_usage if not isinstance(own_cost, (int, float)): - return + return combined_usage self.model_call_details["response_cost"] = own_cost + partial_cost if self.cost_breakdown is not None: self.cost_breakdown = self.cost_breakdown | CostBreakdown( @@ -2674,6 +2685,35 @@ class Logging(LiteLLMLoggingBaseClass): "mid_stream_partial_cost": partial_cost, }, ) + return combined_usage + + def _add_failed_attempts_spend(self, own_usage: Usage | None) -> None: + own_cost: Final = self.model_call_details.get("response_cost") + if own_usage is None or not isinstance(own_cost, (int, float)): + return + failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python( + ( + *self.model_call_details.get("failed_attempts_billed_on_success", ()), + *self.model_call_details.pop("failed_attempts_owed_on_success", ()), + ) + ) + if not failed_attempts: + return + failed_attempts_cost: Final = sum(cost for _, cost in failed_attempts) + self.model_call_details.pop("failed_attempts_usage", None) + self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts + self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)] + ) + self.model_call_details["response_cost"] = own_cost + failed_attempts_cost + if self.cost_breakdown is not None: + self.cost_breakdown = self.cost_breakdown | CostBreakdown( + total_cost=own_cost + failed_attempts_cost, + additional_costs={ + **self.cost_breakdown.get("additional_costs", {}), + "failed_attempts_cost": failed_attempts_cost, + }, + ) def claim_stream_failure_report(self) -> bool: if self._stream_failure_claimed: @@ -2681,13 +2721,22 @@ class Logging(LiteLLMLoggingBaseClass): self._stream_failure_claimed = True return True - def record_earlier_round_for_failure(self, response: object) -> None: + def served_round(self, response: object) -> ServedRound | None: usage: Final = get_usage_object(completion_response=response) - if usage is not None: - earlier_round_price: Final = self._price_without_recording( - response, cache_hit=None, ignore_stamped_cost=False - ) - self.record_partial_usage_for_failure(usage, earlier_round_price.cost or 0.0) + if usage is None: + return None + price: Final = self._price_without_recording(response, cache_hit=None, ignore_stamped_cost=False) + return usage, price.cost or 0.0 + + def record_served_rounds_for_failure(self, served_rounds: tuple[ServedRound, ...]) -> None: + if not served_rounds: + return + self.record_partial_usage_for_failure( + served_rounds[0][0] + if len(served_rounds) == 1 + else BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[usage for usage, _ in served_rounds]), + sum(cost for _, cost in served_rounds), + ) def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" @@ -2935,6 +2984,7 @@ class Logging(LiteLLMLoggingBaseClass): elif "response_cost" in hidden_params: self.model_call_details["response_cost"] = hidden_params["response_cost"] self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) + self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and ( existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped ): @@ -2944,6 +2994,7 @@ class Logging(LiteLLMLoggingBaseClass): pass else: self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result) + self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) if not build_logging_payload: return None @@ -3449,7 +3500,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = self.response_cost_calculator( result=complete_streaming_response ) - self._add_continued_partial_spend(complete_streaming_response) + self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response)) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) logged_streaming_response: Final = self._with_continued_response_id( self._with_client_facing_stream_model(complete_streaming_response) @@ -4041,7 +4092,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model, ) self.model_call_details["response_cost"] = None - self._add_continued_partial_spend(complete_streaming_response) + self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response)) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 762f552dadc..27993d439cd 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -53,6 +53,7 @@ from litellm.constants import ( ) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.otel.runtime import phase_event +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.bug_report import ( allowlisted, bug_report_notice, @@ -129,6 +130,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash +from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( endpoint_unsupported_by_model, @@ -4342,6 +4344,7 @@ class ProxyBaseLLMRequestProcessing: """Raises ProxyException (OpenAI API compatible) if an exception is raised""" log_llm_api_exception(e, self.litellm_call_id) billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj") + failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e) # Allow callbacks to transform the error response transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -4378,7 +4381,9 @@ class ProxyBaseLLMRequestProcessing: model_id=model_id, version=version, response_cost=( - _litellm_logging_obj.finalized_cost.amount + failure_row_response_cost(self.data) + if failed_in_agentic_follow_up + else _litellm_logging_obj.finalized_cost.amount if isinstance(_litellm_logging_obj, LiteLLMLoggingObj) and _litellm_logging_obj.finalized_cost is not None else 0 diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 0d333d4fd08..5781170a47d 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -218,6 +218,32 @@ def _guardrail_info_lifted_from_litellm_metadata( return {"standard_logging_guardrail_information": guardrail_info} +def _failure_row_guardrail_information(request_data: Mapping[str, object]) -> object: + metadata: Final = _str_keyed_fields(request_data.get("metadata")) + lifted: Final = _guardrail_info_lifted_from_litellm_metadata( + _str_keyed_fields(request_data.get("litellm_metadata")), + guardrail_info_already_set="standard_logging_guardrail_information" in metadata, + ) + return {**metadata, **lifted}.get("standard_logging_guardrail_information") + + +def failure_row_response_cost(request_data: Mapping[str, object]) -> float: + stashed_cost: Final = request_data.get("response_cost") + recovered_stream_cost: Final = ( + max(float(stashed_cost), 0.0) + if isinstance(request_data.get("combined_usage_object"), litellm.Usage) + and isinstance(stashed_cost, (int, float)) + else 0.0 + ) + finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object")) + finalized_amount: Final = finalized_failure.get("response_cost") + return ( + float(finalized_amount) + if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float)) + else recovered_stream_cost + guardrail_information_cost(_failure_row_guardrail_information(request_data)) + ) + + def _redaction_opt_in_lifted_from_litellm_metadata( litellm_metadata: ObjectMapping, *, caller_set_headers: bool ) -> ObjectMapping: @@ -543,19 +569,7 @@ class _ProxyDBLogger(CustomLogger): # recovered cost onto request_data (the usage rides along in # ``combined_usage_object`` for the token columns), so attribute the # real partial spend to this failure row instead of zero. - recovered_stream_cost: Final = ( - max(float(request_data.get("response_cost") or 0.0), 0.0) - if isinstance(request_data.get("combined_usage_object"), litellm.Usage) - else 0.0 - ) - finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object")) - finalized_amount: Final = finalized_failure.get("response_cost") - recovered_response_cost: Final = ( - float(finalized_amount) - if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float)) - else recovered_stream_cost - + guardrail_information_cost(existing_metadata.get("standard_logging_guardrail_information")) - ) + recovered_response_cost: Final = failure_row_response_cost(request_data) failure_snapshot_with_late_guardrails: Final = _failure_snapshot_with_guardrails_recorded_after_it( request_data.get("standard_logging_object"), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3e905363102..ddaa8d2e69d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1359,16 +1359,16 @@ def _failure_usage_to_lift( base_model: str | None = None, dispatched_call_type: str | None = None, ) -> tuple[object, object] | None: - failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) current_attempt: Final = _current_failure_usage_to_lift( model_call_details, request_body, - dispatched and (provider_billed or not failed_attempts), + dispatched, provider_billed, deployment_id, base_model, dispatched_call_type=dispatched_call_type, ) + failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) if not failed_attempts: return current_attempt current_usage, current_cost = current_attempt if current_attempt is not None else (None, None) diff --git a/litellm/router.py b/litellm/router.py index 14879ae78b5..6c3777ee40d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -106,6 +106,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.routing import routing_decision_attributes from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.batch_utils import ( batch_create_routing, @@ -3902,6 +3903,8 @@ class Router(DeprecatedRouterMembers): if continuation_output is not None else None ) + if cut_partial is not None: + model_response.logging_obj.mark_stash_billed_by_continuation() if continue_after_content: emitted_tokens: Final = int( getattr(complete_response_object_usage, "completion_tokens", 0) or 0 @@ -7123,6 +7126,8 @@ class Router(DeprecatedRouterMembers): last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see for attempt in range(attempted_retries_for_request(initial_kwargs), budget): trigger = _mid_stream_retry_trigger(last_error) + if is_agentic_followup_failure(trigger): + return _StreamRetriesExhausted(last_error) if not policy_applies and not self._stream_should_retry( trigger, healthy_deployments, all_deployments, initial_kwargs ): @@ -9339,7 +9344,7 @@ class Router(DeprecatedRouterMembers): response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None) return response except Exception as e: - if is_guardrail_intervention(e): + if is_guardrail_intervention(e) or is_agentic_followup_failure(e): raise current_attempt = None original_exception = e @@ -9446,6 +9451,8 @@ class Router(DeprecatedRouterMembers): return response except Exception as e: + if is_agentic_followup_failure(e): + raise # Always track the latest error so we raise the most # recent exception instead of the first one. original_exception = e diff --git a/litellm/utils.py b/litellm/utils.py index 2ca8a49f968..088a5a5f7ce 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -83,6 +83,7 @@ from litellm.constants import ( PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.core_helpers import ( bind_budget_reservation_to_callbacks, normalize_drop_params, @@ -2235,7 +2236,7 @@ def client(original_function): is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_completion_litellm_router_call + num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2269,7 +2270,7 @@ def client(original_function): is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_responses_litellm_router_call + num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2554,7 +2555,10 @@ def client(original_function): is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) if ( - num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError) + num_retries + and not is_acompletion_litellm_router_call + and not isinstance(e, ImportError) + and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops @@ -2585,7 +2589,10 @@ def client(original_function): is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) if ( - num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError) + num_retries + and not is_aresponses_litellm_router_call + and not isinstance(e, ImportError) + and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index ac3707adf13..0a5afc1ecd2 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -22,6 +22,7 @@ they stay out of the body even without it. import datetime import json +from collections.abc import Mapping, Sequence from typing import Any, Dict, Final, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch @@ -264,8 +265,6 @@ class _GateOnlyLogger(CustomLogger): class _RecordingGateLogger(_GateOnlyLogger): - """Also keeps every follow-up response the loop hands to the post hook.""" - def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None: super().__init__(plan=plan, tool_calls=tool_calls) self.followups: tuple[ModelResponse, ...] = () @@ -622,6 +621,35 @@ async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clien assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] +@pytest.mark.asyncio +async def test_refused_rerun_drops_a_call_claimed_only_by_name_next_to_one_claimed_by_id(restore_callbacks): + response: Final = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + _tool_call("call_abc", "litellm_code_execution"), + _tool_call("call_search", "litellm_web_search"), + _tool_call("call_client", "lookup"), + ], + ), + ) + ] + ) + + result: Final = await _run_refused_rerun( + response, + _REFUSING_RAILS[0], + gate_tool_calls={"tool_calls": [{"id": "call_abc"}]}, + other_hooks_tool_calls=({"tool_calls": [{"name": "litellm_web_search"}]},), + ) + + assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] + + @pytest.mark.asyncio @pytest.mark.parametrize( "response, gate_tool_calls", @@ -716,6 +744,118 @@ async def test_acompletion_bills_the_first_call_on_the_failure_log_when_the_foll ) +def _tool_call_turn(call_id: str, usage: Usage) -> ModelResponse: + return ModelResponse( + model="gpt-4o-mini", + choices=[ + Choices( + finish_reason="tool_calls", + message=Message(role="assistant", content=None, tool_calls=[_tool_call(call_id, "litellm_web_search")]), + ) + ], + usage=usage, + ) + + +class _TwoRoundLogger(CustomLogger): + def __init__(self, middle_round: ModelResponse) -> None: + super().__init__() + self._follow_ups: Final = (middle_round, "litellm.RateLimitError") + self.logging_objs: tuple[object, ...] = () + + async def async_should_run_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + assert isinstance(response, ModelResponse) + calls: Final = response.choices[0].message.tool_calls or [] + return bool(calls), {"tool_calls": [{"id": call.id} for call in calls]} + + async def async_build_agentic_loop_plan( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> AgenticLoopPlan: + follow_up: Final = self._follow_ups[len(self.logging_objs)] + self.logging_objs = (*self.logging_objs, logging_obj) + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": follow_up}), + ) + + +@pytest.mark.asyncio +async def test_acompletion_bills_every_served_round_on_the_failure_log_when_a_later_follow_up_fails( + restore_callbacks, monkeypatch: pytest.MonkeyPatch +): + first_usage: Final = Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8) + middle_usage: Final = Usage(prompt_tokens=11, completion_tokens=4, total_tokens=15) + logger: Final = _TwoRoundLogger(middle_round=_tool_call_turn("call_next", middle_usage)) + monkeypatch.setattr(litellm, "callbacks", [logger]) + mock_client: Final = MagicMock() + mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_turn("call_abc", first_usage))) + + with pytest.raises(litellm.RateLimitError): + await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + api_key="sk-test", + client=mock_client, + num_retries=0, + ) + + first_logging_obj, _ = logger.logging_objs + assert isinstance(first_logging_obj, Logging) + price: Final = litellm.model_cost["gpt-4o-mini"] + billed_usage: Final = first_logging_obj.model_call_details["combined_usage_object"] + assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (5 + 11, 3 + 4) + assert first_logging_obj.model_call_details["response_cost"] == pytest.approx( + (5 + 11) * price["input_cost_per_token"] + (3 + 4) * price["output_cost_per_token"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +async def test_a_follow_up_failing_with_a_builtin_error_reaches_the_caller_unchanged(stream: bool): + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + + sandbox_error: Final = RuntimeError("sandbox crashed") + + async def failing_follow_up() -> ModelResponse: + raise sandbox_error + + with pytest.raises(RuntimeError) as raised: + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=Logging( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + stream=stream, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-builtin-error", + function_id="fn-builtin-error", + ), + response=_tool_call_turn("call_abc", Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)), + stream=stream, + ) + + assert raised.value is sandbox_error + + class _RunOnlyLoopLogger(CustomLogger): def __init__(self, result: ModelResponse) -> None: super().__init__() diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 3d6069a3e80..10ae5ed639e 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5,7 +5,7 @@ import json import logging import threading import time -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, Mock, patch @@ -5369,12 +5369,12 @@ class _PlannedChatLoop(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: claimed: Final = [ {"id": call.id, "name": call.function.name} for call in _tool_calls_of(response) @@ -5384,14 +5384,14 @@ class _PlannedChatLoop(CustomLogger): async def async_build_chat_completion_agentic_loop_plan( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> AgenticLoopPlan: self.logging_objs.append(logging_obj) return self._plan @@ -5406,24 +5406,24 @@ class _RunOnlyChatLoop(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: return True, {"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]} async def async_run_chat_completion_agentic_loop( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> object: return self._result @@ -5599,12 +5599,12 @@ class _SearchAgainLoop(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: searches: Final = [ {"id": call.id, "name": call.function.name} for call in response.choices[0].message.tool_calls or [] @@ -5614,14 +5614,14 @@ class _SearchAgainLoop(CustomLogger): async def async_build_chat_completion_agentic_loop_plan( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> AgenticLoopPlan: return AgenticLoopPlan( run_agentic_loop=True, diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 0cb955dd356..0adfe179f01 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -3,7 +3,7 @@ import gzip import importlib import json import os -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -236,12 +236,12 @@ class _SearchingChatGate(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: return True, {"tool_calls": [{"id": "call-1"}]} @@ -253,14 +253,14 @@ class _FailingFollowUpChatLoop(_SearchingChatGate): async def async_run_chat_completion_agentic_loop( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> object: self.runs += 1 self.logging_objs.append(logging_obj) @@ -272,26 +272,26 @@ class _RaisingAgenticGate(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: raise ValueError("gate bug") class _AnsweringChatLoop(_SearchingChatGate): async def async_run_chat_completion_agentic_loop( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> object: self.runs += 1 return litellm.ModelResponse( @@ -490,12 +490,12 @@ class _SearchAgainLoop(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: searches: Final = [ {"id": call.id, "name": call.function.name} for call in response.choices[0].message.tool_calls or [] @@ -505,14 +505,14 @@ class _SearchAgainLoop(CustomLogger): async def async_build_chat_completion_agentic_loop_plan( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> AgenticLoopPlan: return AgenticLoopPlan( run_agentic_loop=True, diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index a7596b51e3e..bdeea581c1c 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -4292,6 +4292,62 @@ class TestHandleLLMApiExceptionPricesWhatTheServerDispatched: assert resent >= dispatched_after_trim +class TestHandleLLMApiExceptionReportsWhatAFailedAgenticFollowUpBills: + async def _error_response_cost(self, raise_the_follow_up_error: bool) -> float: + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + from litellm.types.utils import ModelResponse, Usage + + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={"model": "gpt-4o", "messages": [{"role": "user", "content": "what is 6*7?"}]} + ) + caller: Final = ProxyUserAPIKeyAuth(request_route="/v1/chat/completions") + mock_request: Final = MagicMock(spec=Request) + mock_request.headers = {} + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging.alert_types = [] + _, logging_obj = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=caller, + proxy_logging_obj=proxy_logging, + proxy_config=MagicMock(spec=ProxyConfig), + route_type="acompletion", + ) + logging_obj.model_call_details.update(first_api_call_start_time=datetime.datetime.now()) + follow_up_error: Final = litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o") + + async def failing_follow_up() -> ModelResponse: + raise follow_up_error + + with pytest.raises(litellm.RateLimitError): + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=logging_obj, + response=ModelResponse( + model="gpt-4o", usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672) + ), + stream=False, + ) + with pytest.raises(ProxyException) as raised: + await processing_obj.handle_llm_api_exception( + e=follow_up_error + if raise_the_follow_up_error + else litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o"), + user_api_key_dict=caller, + proxy_logging_obj=proxy_logging, + ) + return float(raised.value.headers["x-litellm-response-cost"]) + + @pytest.mark.asyncio + async def test_the_error_response_reports_the_served_first_call_its_failure_row_bills(self) -> None: + price: Final = litellm.model_cost["gpt-4o"] + + assert await self._error_response_cost(raise_the_follow_up_error=True) == pytest.approx( + 559 * price["input_cost_per_token"] + 113 * price["output_cost_per_token"] + ) + assert await self._error_response_cost(raise_the_follow_up_error=False) == 0 + + @pytest.mark.asyncio class TestHandleLLMApiExceptionDictDetail: """ diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 291231a4aaf..293116d575f 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -3561,13 +3561,15 @@ def test_an_earlier_timed_out_responses_attempt_counts_its_input_from_the_newest [None, 1700000001.0], ids=["retry-not-dispatched", "retry-dispatched"], ) -def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attempts( +def test_a_retry_failure_bills_input_only_for_dispatched_attempts( current_attempt_api_call_start_time: float | None, ) -> None: - from litellm.proxy.utils import _failure_fields_to_lift + from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift from litellm.types.utils import Usage prior_usage: Final = Usage(prompt_tokens=4, completion_tokens=2, total_tokens=6) + estimated_retry_usage: Final = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None) + assert estimated_retry_usage is not None lifted: Final = _failure_fields_to_lift( { @@ -3586,43 +3588,63 @@ def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attemp usage: Final = lifted["combined_usage_object"] assert isinstance(usage, Usage) - assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (4, 2, 6) + expected_retry_tokens: Final = ( + estimated_retry_usage.prompt_tokens if current_attempt_api_call_start_time is not None else 0 + ) + assert usage.prompt_tokens == prior_usage.prompt_tokens + expected_retry_tokens + assert usage.completion_tokens == prior_usage.completion_tokens + assert usage.total_tokens == prior_usage.total_tokens + expected_retry_tokens assert lifted["response_cost"] == 0.25 -def test_a_responses_failure_after_a_retried_follow_up_bills_only_the_first_calls_usage() -> None: - from litellm.litellm_core_utils.litellm_logging import Logging +@pytest.mark.asyncio +async def test_an_agentic_follow_up_failure_row_bills_each_served_first_call_without_an_estimate() -> None: + import litellm + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure from litellm.proxy.utils import _failure_fields_to_lift from litellm.types.utils import ModelResponse, Usage - responses_input: Final = [{"role": "user", "content": "Search the web for the latest LiteLLM release"}] logging_obj: Final = Logging( model=FAILURE_USAGE_MODEL, - messages=responses_input, + messages=ONE_USER_MESSAGE, stream=False, - call_type="aresponses", + call_type="acompletion", start_time=real_datetime.datetime.now(), - litellm_call_id="call-responses-retry", - function_id="fn-responses-retry", + litellm_call_id="call-agentic-follow-up", + function_id="fn-agentic-follow-up", ) logging_obj.update_environment_variables( model=FAILURE_USAGE_MODEL, user="", optional_params={}, litellm_params={"custom_llm_provider": "openai"} ) - logging_obj.pre_call(input=responses_input, api_key="sk-test") - first_call: Final = ModelResponse(usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672)) - logging_obj.record_earlier_round_for_failure(first_call) - first_call_cost: Final = logging_obj.model_call_details["response_cost"] - logging_obj.carry_failed_attempt_usage() - logging_obj.pre_call(input=responses_input, api_key="sk-test") - lifted: Final = _failure_fields_to_lift( - {"litellm_logging_obj": logging_obj}, provider_billed=False, dispatched_call_type="aresponses" + async def failing_follow_up() -> ModelResponse: + raise litellm.RateLimitError(message="rate limited", llm_provider="openai", model=FAILURE_USAGE_MODEL) + + async def serve_first_call_then_fail_its_follow_up(usage: Usage) -> float: + logging_obj.pre_call(input=ONE_USER_MESSAGE, api_key="sk-test") + with pytest.raises(litellm.RateLimitError): + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=logging_obj, + response=ModelResponse(model=FAILURE_USAGE_MODEL, usage=usage), + stream=False, + ) + return logging_obj.model_call_details["response_cost"] + + primary_cost: Final = await serve_first_call_then_fail_its_follow_up( + Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672) + ) + logging_obj.carry_failed_attempt_usage() + backup_cost: Final = await serve_first_call_then_fail_its_follow_up( + Usage(prompt_tokens=31, completion_tokens=5, total_tokens=36) ) + lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}) + usage: Final = lifted["combined_usage_object"] - assert (usage.prompt_tokens, usage.completion_tokens) == (559, 113) - assert first_call_cost > 0 - assert lifted["response_cost"] == pytest.approx(first_call_cost) + assert (usage.prompt_tokens, usage.completion_tokens) == (559 + 31, 113 + 5) + assert primary_cost > 0 and backup_cost > 0 + assert lifted["response_cost"] == pytest.approx(primary_cost + backup_cost) @pytest.mark.parametrize( diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index af9a0b50805..07efd201523 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -26,7 +26,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.xai.common_utils import xai_reported_cost_in_usd from litellm.proxy.mcp_server import mcp_server_manager from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk -from litellm.proxy.utils import _failure_fields_to_lift +from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator from litellm.types.llms.openai import ( @@ -540,13 +540,9 @@ async def test_mcp_allowed_tools_filtering(): ] # First filter by allowed tools - filtered_tools_with_duplicates = ( - LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( - mcp_tools=mock_mcp_tools_with_duplicates, - mcp_tools_with_litellm_proxy=cast( - List[ToolParam], mcp_tool_config_with_duplicates - ), - ) + filtered_tools_with_duplicates = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( + mcp_tools=mock_mcp_tools_with_duplicates, + mcp_tools_with_litellm_proxy=cast(List[ToolParam], mcp_tool_config_with_duplicates), ) # Then deduplicate the filtered tools @@ -605,11 +601,9 @@ async def test_mcp_allowed_tools_filtering(): }, ] - filtered_tools_multiple = ( - LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( - mcp_tools=mock_mcp_tools_from_server, - mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs), - ) + filtered_tools_multiple = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( + mcp_tools=mock_mcp_tools_from_server, + mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs), ) # Should return union of all allowed tools (3 unique tools) @@ -1918,7 +1912,7 @@ def _router_retrying_once() -> litellm.Router: @pytest.mark.asyncio -async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds( +async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): _serve_responses( @@ -1945,9 +1939,11 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_ea assert len(recorder.logged) == 1 row: Final = recorder.logged[0] assert row["id"] == response.id - assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5) - assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens) - assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY)) + assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5) + assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19) + assert row["response_cost"] == pytest.approx( + _round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY) + ) @pytest.mark.asyncio @@ -1982,7 +1978,22 @@ async def test_non_stream_auto_execute_failing_every_retry_bills_each_paid_first failure_row_fields: Final = _failure_fields_to_lift(MappingProxyType({"litellm_logging_obj": logging_obj})) billed_usage: Final = failure_row_fields["combined_usage_object"] assert isinstance(billed_usage, Usage) - assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == billed_tokens + estimated_failure_usage: Final = ( + _estimate_dispatched_failure_usage( + model="gpt-5.6", + request_input=logging_obj.model_call_details.get("messages"), + system_input=None, + ) + if not retried_first_rounds + else None + ) + expected_prompt_tokens: Final = billed_tokens[0] + ( + estimated_failure_usage.prompt_tokens if estimated_failure_usage is not None else 0 + ) + assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == ( + expected_prompt_tokens, + billed_tokens[1], + ) assert failure_row_fields["response_cost"] == pytest.approx( _round_cost(_ROUND_ONE_BODY) + sum(_round_cost(body) for body in retried_first_rounds) ) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index dc7e8d64889..94708f5841a 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -3652,14 +3652,28 @@ class _CutSSE(httpx.AsyncByteStream): raise httpx.RemoteProtocolError("peer closed connection without sending complete message body") -_BACKUP_SSE: Final = "".join( - f"data: {json.dumps(chunk)}\n\n" - for chunk in ( - {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", - "choices": [{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}]}, - {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", - "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}}, +_BACKUP_SSE: Final = ( + "".join( + f"data: {json.dumps(chunk)}\n\n" + for chunk in ( + { + "id": "b", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "mistral-large-latest", + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None} + ], + }, + { + "id": "b", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "mistral-large-latest", + "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}, + }, + ) ) ) + "data: [DONE]\n\n" @@ -3907,88 +3921,165 @@ class _ScriptedAgenticFollowUps(CustomLogger): self.shared_call_succeeded.set() -async def _route_agentic_request_with_retries( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...] +_AGENTIC_BACKUP_USAGE: Final = {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23} +_AGENTIC_BACKUP_BODY: Final = { + "id": "chatcmpl-backup", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "42"}}], + "usage": _AGENTIC_BACKUP_USAGE, +} +_AGENTIC_RATE_LIMITED: Final = { + "error": {"message": "Rate limit reached", "type": "requests", "code": "rate_limit_exceeded"} +} + + +async def _route_agentic_request( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + follow_ups: tuple[str, ...], + model: str = "m", + first_call_replies: tuple[httpx.Response, ...] = (), + fallbacks: tuple[str, ...] = (), ) -> tuple[object, LiteLLMLogging, respx.Route, _ScriptedAgenticFollowUps]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) hook: Final = _ScriptedAgenticFollowUps(follow_ups) monkeypatch.setattr(litellm, "callbacks", [hook]) - first_calls: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( - return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY) + first_calls: Final = ( + respx_mock.post("https://api.openai.com/v1/chat/completions").mock(side_effect=list(first_call_replies)) + if first_call_replies + else respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY) + ) + ) + respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_AGENTIC_BACKUP_BODY) ) router: Final = Router( - model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}], + model_list=[ + {"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}, + { + "model_name": "backup", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-test", + "api_base": "https://backup.test/v1", + }, + }, + ], num_retries=2, retry_after=0, + fallbacks=[{"m": list(fallbacks)}] if fallbacks else [], ) messages: Final = [{"role": "user", "content": "what is 6*7?"}] shared_logging, _ = litellm.utils.function_setup( original_function="acompletion", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), - model="m", + model=model, messages=messages, litellm_call_id="shared-agentic-call", ) try: outcome: object = await router.acompletion( - model="m", messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging + model=model, messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging ) except litellm.RateLimitError as error: outcome = error return outcome, shared_logging, first_calls, hook +def _gpt_4o_mini_cost(usage: Mapping[str, int]) -> float: + return sum( + litellm.cost_per_token( + model="gpt-4o-mini", prompt_tokens=usage["prompt_tokens"], completion_tokens=usage["completion_tokens"] + ) + ) + + @pytest.mark.asyncio -async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_first_call( +async def test_agentic_request_whose_follow_up_fails_is_not_retried_and_bills_its_one_first_call( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter ) -> None: from litellm.proxy.utils import _failure_fields_to_lift - outcome, shared_logging, first_calls, _ = await _route_agentic_request_with_retries( + outcome, shared_logging, first_calls, _ = await _route_agentic_request( monkeypatch, respx_mock, ("litellm.RateLimitError",) ) lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": shared_logging}) - first_call_cost: Final = sum( - litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7) - ) assert isinstance(outcome, litellm.RateLimitError) - assert first_calls.call_count == 3 - assert lifted["combined_usage_object"].prompt_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] - assert lifted["combined_usage_object"].completion_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] - assert lifted["response_cost"] == pytest.approx(3 * first_call_cost) - - -async def _successful_agentic_request_cost_report( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...] -) -> tuple[int, Mapping[str, object], Mapping[str, str | None]]: - from litellm.proxy.common_request_processing import cost_headers - - outcome, shared_logging, _, hook = await _route_agentic_request_with_retries(monkeypatch, respx_mock, follow_ups) - await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) - assert isinstance(outcome, litellm.ModelResponse) - success: Final = shared_logging.model_call_details["standard_logging_object"] - billed: Final = { - field: success[field] for field in ("response_cost", "prompt_tokens", "completion_tokens", "cost_breakdown") - } - return hook.plans_built, billed, cost_headers(success["response_cost"], shared_logging) + assert outcome.status_code == 429 + assert first_calls.call_count == 1 + assert lifted["combined_usage_object"].prompt_tokens == _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] + assert lifted["combined_usage_object"].completion_tokens == _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] + assert lifted["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE)) @pytest.mark.asyncio -async def test_retried_agentic_request_that_recovers_reports_the_cost_of_a_first_try_success( +async def test_agentic_request_whose_first_call_fails_is_still_retried( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter ) -> None: - first_try_attempts, first_try_billed, first_try_headers = await _successful_agentic_request_cost_report( - monkeypatch, respx_mock, ("42",) - ) - recovered_attempts, recovered_billed, recovered_headers = await _successful_agentic_request_cost_report( - monkeypatch, respx_mock, ("litellm.RateLimitError", "42") + outcome, _, first_calls, hook = await _route_agentic_request( + monkeypatch, + respx_mock, + ("42",), + first_call_replies=( + httpx.Response(429, json=_AGENTIC_RATE_LIMITED), + httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY), + ), ) - assert (first_try_attempts, recovered_attempts) == (1, 2) - assert recovered_billed == first_try_billed - assert recovered_headers == first_try_headers + assert isinstance(outcome, litellm.ModelResponse) + assert first_calls.call_count == 2 + assert hook.plans_built == 1 + + +async def _successful_agentic_request_billing( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + follow_ups: tuple[str, ...], + model: str, + fallbacks: tuple[str, ...] = (), +) -> tuple[Mapping[str, object], Mapping[str, str | None]]: + from litellm.proxy.common_request_processing import cost_headers + + outcome, shared_logging, _, hook = await _route_agentic_request( + monkeypatch, respx_mock, follow_ups, model=model, fallbacks=fallbacks + ) + await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) + assert isinstance(outcome, litellm.ModelResponse) + success: Final = shared_logging.model_call_details["standard_logging_object"] + return success, cost_headers(success["response_cost"], shared_logging) + + +@pytest.mark.asyncio +async def test_fallback_after_a_failed_agentic_follow_up_bills_the_primarys_first_call_on_the_success_row( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + first_try, first_try_headers = await _successful_agentic_request_billing( + monkeypatch, respx_mock, ("42",), model="backup" + ) + recovered, recovered_headers = await _successful_agentic_request_billing( + monkeypatch, respx_mock, ("litellm.RateLimitError",), model="m", fallbacks=("backup",) + ) + + first_call_cost: Final = _gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE) + assert first_try["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_BACKUP_USAGE)) + assert "failed_attempts_cost" not in first_try["cost_breakdown"].get("additional_costs", {}) + assert recovered["response_cost"] == pytest.approx(first_try["response_cost"] + first_call_cost) + assert recovered["prompt_tokens"] == first_try["prompt_tokens"] + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] + assert ( + recovered["completion_tokens"] + == first_try["completion_tokens"] + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] + ) + assert recovered["cost_breakdown"]["total_cost"] == pytest.approx(recovered["response_cost"]) + assert recovered["cost_breakdown"]["additional_costs"]["failed_attempts_cost"] == pytest.approx(first_call_cost) + assert float(recovered_headers["x-litellm-response-cost"]) == pytest.approx(recovered["response_cost"]) + assert {key: value for key, value in recovered_headers.items() if key != "x-litellm-response-cost"} == { + key: value for key, value in first_try_headers.items() if key != "x-litellm-response-cost" + } def _responses_sse(*events: Mapping[str, object]) -> str: @@ -13817,7 +13908,11 @@ def test_get_search_tool_support_reports_each_routable_deployment_of_the_group() @pytest.mark.parametrize("provider", ["chatgpt", "github_copilot"]) @pytest.mark.parametrize("explicit_provider", [False, True]) async def test_search_tool_catalog_lookup_needs_no_credentials( - provider: str, explicit_provider: bool, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + provider: str, + explicit_provider: bool, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, ) -> None: monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "chatgpt")) monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path / "copilot")) @@ -17485,13 +17580,16 @@ def test_model_group_info_fast_mode_fails_closed_without_explicit_boolean( def test_model_group_info_fast_mode_requires_every_deployment( local_model_cost_map: None, other_model: str, expected: bool, reverse: bool ) -> None: - models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ( - "anthropic/claude-opus-5", other_model + models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ("anthropic/claude-opus-5", other_model) + router: Final = Router( + model_list=[ + { + "model_name": "fast-group", + "litellm_params": {"model": model, "api_key": "fake-key"}, + } + for model in models + ] ) - router: Final = Router(model_list=[{ - "model_name": "fast-group", - "litellm_params": {"model": model, "api_key": "fake-key"}, - } for model in models]) result: Final = router.get_model_group_info("fast-group") @@ -17849,10 +17947,7 @@ class _AnthropicMessagesFallbackByteStream: def _anthropic_messages_overloaded_error_chunk() -> bytes: - return ( - b"event: error\n" - b'data: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n' - ) + return b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n' def _anthropic_messages_invalid_request_error_chunk() -> bytes: @@ -24600,16 +24695,25 @@ class TestAutoRouterTraceProvenance: monkeypatch.setattr(litellm, "_async_success_callback", [logger, CallbackComplete()]) classifier: Final = ObservedClassifier() - router: Final = Router(model_list=[ - {**self._marker("blue"), "litellm_params": { - "model": "auto_router/complexity_router", - "complexity_router_config": { - "classifier_type": "custom", "classifier_plugin": classifier, - "tiers": {"SIMPLE": "answer"}, "session_affinity": False, "deployment_affinity": False, + router: Final = Router( + model_list=[ + { + **self._marker("blue"), + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "custom", + "classifier_plugin": classifier, + "tiers": {"SIMPLE": "answer"}, + "session_affinity": False, + "deployment_affinity": False, + }, + }, }, - }}, - {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, - ], plugins=[classifier]) + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ], + plugins=[classifier], + ) stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"} request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale, "user_api_key_team_id": "team"}} with logger.tracer.start_as_current_span("request") as root: @@ -24694,20 +24798,22 @@ class TestAutoRouterTraceProvenance: self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]" ) -> None: logger, exporter = _tracing - router: Final = Router(model_list=[ - self._marker("blue"), - { - "model_name": "traced-router", - "model_info": {"id": "definition-semantic"}, - "litellm_params": { - "model": "auto_router/semantic-router", - "tags": ["blue"], - "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), - "auto_router_default_model": "answer", - "auto_router_embedding_model": "embed", + router: Final = Router( + model_list=[ + self._marker("blue"), + { + "model_name": "traced-router", + "model_info": {"id": "definition-semantic"}, + "litellm_params": { + "model": "auto_router/semantic-router", + "tags": ["blue"], + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + }, }, - }, - ]) + ] + ) with logger.start_phase_span("route traced-router"): response: Final = await router.async_pre_routing_hook( model="traced-router", request_kwargs={"metadata": {"tags": ["blue"]}} @@ -24727,26 +24833,42 @@ class TestAutoRouterTraceProvenance: ) -> None: logger, exporter = _tracing monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - respx_mock.post("https://embedding.example/v1/embeddings").respond(200, json={ - "object": "list", - "model": "text-embedding-3-small", - "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], - "usage": {"prompt_tokens": 1, "total_tokens": 1}, - }) - router: Final = Router(model_list=[ - { - "model_name": "internal-semantic-router", - "model_info": {"id": "semantic-definition", "team_id": "trace-team", "team_public_model_name": "public-router"}, - "litellm_params": { - "model": "auto_router/semantic-router", - "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), - "auto_router_default_model": "answer", - "auto_router_embedding_model": "embed", - }, + respx_mock.post("https://embedding.example/v1/embeddings").respond( + 200, + json={ + "object": "list", + "model": "text-embedding-3-small", + "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], + "usage": {"prompt_tokens": 1, "total_tokens": 1}, }, - {"model_name": "embed", "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", "api_key": "test"}}, - {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, - ]) + ) + router: Final = Router( + model_list=[ + { + "model_name": "internal-semantic-router", + "model_info": { + "id": "semantic-definition", + "team_id": "trace-team", + "team_public_model_name": "public-router", + }, + "litellm_params": { + "model": "auto_router/semantic-router", + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + }, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_base": "https://embedding.example/v1", + "api_key": "test", + }, + }, + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ] + ) request: Final = {"metadata": {"user_api_key_team_id": "trace-team"}} with logger.tracer.start_as_current_span("request"): await router.async_get_available_deployment(