mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(agentic): stop retrying a request whose agentic follow-up failed and bill every round it served
An error raised by an agentic follow-up after the first call was served now carries a typed mark, so the router's retry loops, the mid-stream retry, and litellm's num_retries wrapper raise it as is instead of rerunning the whole request. A failure of the first call itself still retries, and fallbacks still run. The failure row bills every round the loop served before the failure, at any depth, and a later attempt that succeeds bills what the failed attempts paid on its success row, with the cost breakdown and the response cost headers adding up. The error response's x-litellm-response-cost reports the same amount its failure row bills. Failure rows outside agentic hooks keep the prompt-token estimate they had before. A refused tool call now matches by id among the refused ids, or by name among refused calls that carry no id, so a name-only refusal next to an id refusal is no longer kept.
This commit is contained in:
parent
4df38d14aa
commit
7fd14e5beb
16 changed files with 711 additions and 243 deletions
27
litellm/litellm_core_utils/agentic_followup_failure.py
Normal file
27
litellm/litellm_core_utils/agentic_followup_failure.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import weakref
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
ServedRound: TypeAlias = tuple[Usage, float]
|
||||
|
||||
_followup_failures: Final[weakref.WeakKeyDictionary[BaseException, tuple[ServedRound, ...]]] = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
def _can_be_marked(error: BaseException) -> bool:
|
||||
return type(error).__weakrefoffset__ != 0
|
||||
|
||||
|
||||
def mark_agentic_followup_failure(error: BaseException, served_rounds: tuple[ServedRound, ...]) -> None:
|
||||
if _can_be_marked(error):
|
||||
_followup_failures[error] = served_rounds
|
||||
|
||||
|
||||
def is_agentic_followup_failure(error: BaseException) -> bool:
|
||||
return _can_be_marked(error) and error in _followup_failures
|
||||
|
||||
|
||||
def rounds_served_before(error: BaseException) -> tuple[ServedRound, ...]:
|
||||
return _followup_failures.get(error, ()) if _can_be_marked(error) else ()
|
||||
|
|
@ -40,19 +40,19 @@ async def every_claimed_tool_call(
|
|||
return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))})
|
||||
|
||||
|
||||
def _call_ids(call: object) -> frozenset[str]:
|
||||
return _strings((message_field(call, "id"), message_field(call, "call_id")))
|
||||
|
||||
|
||||
def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
|
||||
calls: Final = _claimed_calls(tool_calls)
|
||||
ids: Final = _strings(
|
||||
chain.from_iterable((message_field(call, "id"), message_field(call, "call_id")) for call in calls)
|
||||
)
|
||||
names: Final = _strings(message_field(call, "name") for call in calls)
|
||||
ids: Final = frozenset(chain.from_iterable(_call_ids(call) for call in calls))
|
||||
names: Final = _strings(message_field(call, "name") for call in calls if not _call_ids(call))
|
||||
return ids, names
|
||||
|
||||
|
||||
def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
|
||||
if isinstance(call_id, str) and refused_ids:
|
||||
return call_id in refused_ids
|
||||
return name in refused_names
|
||||
return (isinstance(call_id, str) and call_id in refused_ids) or name in refused_names
|
||||
|
||||
|
||||
def is_refused_item(
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure, rounds_served_before
|
||||
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
|
||||
from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
|
|
@ -54,9 +55,14 @@ async def bill_first_round_on_failure(
|
|||
) -> _DispatchResult:
|
||||
try:
|
||||
return await dispatch
|
||||
except Exception:
|
||||
if not stream and isinstance(logging_obj, LiteLLMLoggingObject):
|
||||
logging_obj.record_earlier_round_for_failure(response)
|
||||
except Exception as error:
|
||||
if stream or not isinstance(logging_obj, LiteLLMLoggingObject):
|
||||
mark_agentic_followup_failure(error, rounds_served_before(error))
|
||||
raise
|
||||
this_round: Final = logging_obj.served_round(response)
|
||||
served_rounds: Final = (*rounds_served_before(error), *(() if this_round is None else (this_round,)))
|
||||
mark_agentic_followup_failure(error, served_rounds)
|
||||
logging_obj.record_served_rounds_for_failure(served_rounds)
|
||||
raise
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
)
|
||||
from litellm.cost_calculator import (
|
||||
BaseTokenUsageProcessor,
|
||||
RealtimeAPITokenUsageProcessor,
|
||||
ResponsesWebSocketTokenUsageProcessor,
|
||||
get_usage_object,
|
||||
|
|
@ -70,6 +71,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger
|
|||
from litellm.integrations.langtrace import langtrace_trace_endpoint
|
||||
from litellm.integrations.mlflow import MlflowLogger
|
||||
from litellm.integrations.sqs import SQSLogger
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import ServedRound
|
||||
from litellm.litellm_core_utils.classifier_logging import (
|
||||
classifier_audit_fields,
|
||||
classifier_input_snapshot,
|
||||
|
|
@ -571,6 +573,7 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"
|
|||
_LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
|
||||
_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
|
||||
_REDACTED_RESULT: Final = TypeAdapter(object)
|
||||
_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[ServedRound, ...])
|
||||
_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
|
||||
|
||||
|
||||
|
|
@ -2636,35 +2639,43 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
own usage while the request's failure log still bills every attempt."""
|
||||
usage: Final = self._model_call_details_view.get("combined_usage_object")
|
||||
response_cost: Final = self._model_call_details_view.get("response_cost")
|
||||
billed_by_continuation: Final = self.model_call_details.pop("stash_billed_by_continuation", False) is True
|
||||
self.model_call_details.pop("combined_usage_object", None)
|
||||
self.model_call_details.pop("response_cost", None)
|
||||
if isinstance(usage, Usage):
|
||||
self.model_call_details["current_attempt_api_call_start_time"] = None
|
||||
self.model_call_details["failed_attempts_usage"] = (
|
||||
*self.model_call_details.get("failed_attempts_usage", ()),
|
||||
(usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
|
||||
if not isinstance(usage, Usage):
|
||||
return
|
||||
attempt: Final = (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0)
|
||||
self.model_call_details["current_attempt_api_call_start_time"] = None
|
||||
self.model_call_details["failed_attempts_usage"] = (
|
||||
*self.model_call_details.get("failed_attempts_usage", ()),
|
||||
attempt,
|
||||
)
|
||||
if not billed_by_continuation:
|
||||
self.model_call_details["failed_attempts_owed_on_success"] = (
|
||||
*self.model_call_details.get("failed_attempts_owed_on_success", ()),
|
||||
attempt,
|
||||
)
|
||||
|
||||
def mark_stash_billed_by_continuation(self) -> None:
|
||||
self.model_call_details["stash_billed_by_continuation"] = True
|
||||
|
||||
def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
|
||||
"""Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
|
||||
self.model_call_details["continued_partial_usage"] = usage
|
||||
self.model_call_details["continued_partial_cost"] = response_cost
|
||||
self.model_call_details["continued_response_id"] = response_id
|
||||
|
||||
def _add_continued_partial_spend(self, response: object) -> None:
|
||||
def _add_continued_partial_spend(self, response: object) -> Usage | None:
|
||||
partial_usage: Final = self.model_call_details.get("continued_partial_usage")
|
||||
partial_cost: Final = self.model_call_details.get("continued_partial_cost")
|
||||
own_usage: Final = getattr(response, "usage", None)
|
||||
own_cost: Final = self.model_call_details.get("response_cost")
|
||||
if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
|
||||
return
|
||||
from litellm.cost_calculator import BaseTokenUsageProcessor
|
||||
|
||||
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
|
||||
usage_objects=[own_usage, partial_usage]
|
||||
)
|
||||
return own_usage if isinstance(own_usage, Usage) else None
|
||||
combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[own_usage, partial_usage])
|
||||
self.model_call_details["combined_usage_object"] = combined_usage
|
||||
if not isinstance(own_cost, (int, float)):
|
||||
return
|
||||
return combined_usage
|
||||
self.model_call_details["response_cost"] = own_cost + partial_cost
|
||||
if self.cost_breakdown is not None:
|
||||
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
|
||||
|
|
@ -2674,6 +2685,35 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"mid_stream_partial_cost": partial_cost,
|
||||
},
|
||||
)
|
||||
return combined_usage
|
||||
|
||||
def _add_failed_attempts_spend(self, own_usage: Usage | None) -> None:
|
||||
own_cost: Final = self.model_call_details.get("response_cost")
|
||||
if own_usage is None or not isinstance(own_cost, (int, float)):
|
||||
return
|
||||
failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(
|
||||
(
|
||||
*self.model_call_details.get("failed_attempts_billed_on_success", ()),
|
||||
*self.model_call_details.pop("failed_attempts_owed_on_success", ()),
|
||||
)
|
||||
)
|
||||
if not failed_attempts:
|
||||
return
|
||||
failed_attempts_cost: Final = sum(cost for _, cost in failed_attempts)
|
||||
self.model_call_details.pop("failed_attempts_usage", None)
|
||||
self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts
|
||||
self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
|
||||
usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)]
|
||||
)
|
||||
self.model_call_details["response_cost"] = own_cost + failed_attempts_cost
|
||||
if self.cost_breakdown is not None:
|
||||
self.cost_breakdown = self.cost_breakdown | CostBreakdown(
|
||||
total_cost=own_cost + failed_attempts_cost,
|
||||
additional_costs={
|
||||
**self.cost_breakdown.get("additional_costs", {}),
|
||||
"failed_attempts_cost": failed_attempts_cost,
|
||||
},
|
||||
)
|
||||
|
||||
def claim_stream_failure_report(self) -> bool:
|
||||
if self._stream_failure_claimed:
|
||||
|
|
@ -2681,13 +2721,22 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self._stream_failure_claimed = True
|
||||
return True
|
||||
|
||||
def record_earlier_round_for_failure(self, response: object) -> None:
|
||||
def served_round(self, response: object) -> ServedRound | None:
|
||||
usage: Final = get_usage_object(completion_response=response)
|
||||
if usage is not None:
|
||||
earlier_round_price: Final = self._price_without_recording(
|
||||
response, cache_hit=None, ignore_stamped_cost=False
|
||||
)
|
||||
self.record_partial_usage_for_failure(usage, earlier_round_price.cost or 0.0)
|
||||
if usage is None:
|
||||
return None
|
||||
price: Final = self._price_without_recording(response, cache_hit=None, ignore_stamped_cost=False)
|
||||
return usage, price.cost or 0.0
|
||||
|
||||
def record_served_rounds_for_failure(self, served_rounds: tuple[ServedRound, ...]) -> None:
|
||||
if not served_rounds:
|
||||
return
|
||||
self.record_partial_usage_for_failure(
|
||||
served_rounds[0][0]
|
||||
if len(served_rounds) == 1
|
||||
else BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[usage for usage, _ in served_rounds]),
|
||||
sum(cost for _, cost in served_rounds),
|
||||
)
|
||||
|
||||
def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
|
||||
"""Bill a fully streamed response on the failure log when a post-call hook rejects it."""
|
||||
|
|
@ -2935,6 +2984,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
elif "response_cost" in hidden_params:
|
||||
self.model_call_details["response_cost"] = hidden_params["response_cost"]
|
||||
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
|
||||
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
|
||||
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
|
||||
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
|
||||
):
|
||||
|
|
@ -2944,6 +2994,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
pass
|
||||
else:
|
||||
self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result)
|
||||
self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result))
|
||||
|
||||
if not build_logging_payload:
|
||||
return None
|
||||
|
|
@ -3449,7 +3500,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model_call_details["response_cost"] = self.response_cost_calculator(
|
||||
result=complete_streaming_response
|
||||
)
|
||||
self._add_continued_partial_spend(complete_streaming_response)
|
||||
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
|
||||
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
|
||||
logged_streaming_response: Final = self._with_continued_response_id(
|
||||
self._with_client_facing_stream_model(complete_streaming_response)
|
||||
|
|
@ -4041,7 +4092,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.model,
|
||||
)
|
||||
self.model_call_details["response_cost"] = None
|
||||
self._add_continued_partial_spend(complete_streaming_response)
|
||||
self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response))
|
||||
|
||||
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.otel.runtime import phase_event
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
allowlisted,
|
||||
bug_report_notice,
|
||||
|
|
@ -129,6 +130,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
|
|||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.proxy.route_llm_request import (
|
||||
endpoint_unsupported_by_model,
|
||||
|
|
@ -4342,6 +4344,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"""Raises ProxyException (OpenAI API compatible) if an exception is raised"""
|
||||
log_llm_api_exception(e, self.litellm_call_id)
|
||||
billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj")
|
||||
failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e)
|
||||
# Allow callbacks to transform the error response
|
||||
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -4378,7 +4381,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
model_id=model_id,
|
||||
version=version,
|
||||
response_cost=(
|
||||
_litellm_logging_obj.finalized_cost.amount
|
||||
failure_row_response_cost(self.data)
|
||||
if failed_in_agentic_follow_up
|
||||
else _litellm_logging_obj.finalized_cost.amount
|
||||
if isinstance(_litellm_logging_obj, LiteLLMLoggingObj)
|
||||
and _litellm_logging_obj.finalized_cost is not None
|
||||
else 0
|
||||
|
|
|
|||
|
|
@ -218,6 +218,32 @@ def _guardrail_info_lifted_from_litellm_metadata(
|
|||
return {"standard_logging_guardrail_information": guardrail_info}
|
||||
|
||||
|
||||
def _failure_row_guardrail_information(request_data: Mapping[str, object]) -> object:
|
||||
metadata: Final = _str_keyed_fields(request_data.get("metadata"))
|
||||
lifted: Final = _guardrail_info_lifted_from_litellm_metadata(
|
||||
_str_keyed_fields(request_data.get("litellm_metadata")),
|
||||
guardrail_info_already_set="standard_logging_guardrail_information" in metadata,
|
||||
)
|
||||
return {**metadata, **lifted}.get("standard_logging_guardrail_information")
|
||||
|
||||
|
||||
def failure_row_response_cost(request_data: Mapping[str, object]) -> float:
|
||||
stashed_cost: Final = request_data.get("response_cost")
|
||||
recovered_stream_cost: Final = (
|
||||
max(float(stashed_cost), 0.0)
|
||||
if isinstance(request_data.get("combined_usage_object"), litellm.Usage)
|
||||
and isinstance(stashed_cost, (int, float))
|
||||
else 0.0
|
||||
)
|
||||
finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object"))
|
||||
finalized_amount: Final = finalized_failure.get("response_cost")
|
||||
return (
|
||||
float(finalized_amount)
|
||||
if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float))
|
||||
else recovered_stream_cost + guardrail_information_cost(_failure_row_guardrail_information(request_data))
|
||||
)
|
||||
|
||||
|
||||
def _redaction_opt_in_lifted_from_litellm_metadata(
|
||||
litellm_metadata: ObjectMapping, *, caller_set_headers: bool
|
||||
) -> ObjectMapping:
|
||||
|
|
@ -543,19 +569,7 @@ class _ProxyDBLogger(CustomLogger):
|
|||
# recovered cost onto request_data (the usage rides along in
|
||||
# ``combined_usage_object`` for the token columns), so attribute the
|
||||
# real partial spend to this failure row instead of zero.
|
||||
recovered_stream_cost: Final = (
|
||||
max(float(request_data.get("response_cost") or 0.0), 0.0)
|
||||
if isinstance(request_data.get("combined_usage_object"), litellm.Usage)
|
||||
else 0.0
|
||||
)
|
||||
finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object"))
|
||||
finalized_amount: Final = finalized_failure.get("response_cost")
|
||||
recovered_response_cost: Final = (
|
||||
float(finalized_amount)
|
||||
if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float))
|
||||
else recovered_stream_cost
|
||||
+ guardrail_information_cost(existing_metadata.get("standard_logging_guardrail_information"))
|
||||
)
|
||||
recovered_response_cost: Final = failure_row_response_cost(request_data)
|
||||
|
||||
failure_snapshot_with_late_guardrails: Final = _failure_snapshot_with_guardrails_recorded_after_it(
|
||||
request_data.get("standard_logging_object"),
|
||||
|
|
|
|||
|
|
@ -1359,16 +1359,16 @@ def _failure_usage_to_lift(
|
|||
base_model: str | None = None,
|
||||
dispatched_call_type: str | None = None,
|
||||
) -> tuple[object, object] | None:
|
||||
failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ()))
|
||||
current_attempt: Final = _current_failure_usage_to_lift(
|
||||
model_call_details,
|
||||
request_body,
|
||||
dispatched and (provider_billed or not failed_attempts),
|
||||
dispatched,
|
||||
provider_billed,
|
||||
deployment_id,
|
||||
base_model,
|
||||
dispatched_call_type=dispatched_call_type,
|
||||
)
|
||||
failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ()))
|
||||
if not failed_attempts:
|
||||
return current_attempt
|
||||
current_usage, current_cost = current_attempt if current_attempt is not None else (None, None)
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_
|
|||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.otel.routing import routing_decision_attributes
|
||||
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.batch_utils import (
|
||||
batch_create_routing,
|
||||
|
|
@ -3902,6 +3903,8 @@ class Router(DeprecatedRouterMembers):
|
|||
if continuation_output is not None
|
||||
else None
|
||||
)
|
||||
if cut_partial is not None:
|
||||
model_response.logging_obj.mark_stash_billed_by_continuation()
|
||||
if continue_after_content:
|
||||
emitted_tokens: Final = int(
|
||||
getattr(complete_response_object_usage, "completion_tokens", 0) or 0
|
||||
|
|
@ -7123,6 +7126,8 @@ class Router(DeprecatedRouterMembers):
|
|||
last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see
|
||||
for attempt in range(attempted_retries_for_request(initial_kwargs), budget):
|
||||
trigger = _mid_stream_retry_trigger(last_error)
|
||||
if is_agentic_followup_failure(trigger):
|
||||
return _StreamRetriesExhausted(last_error)
|
||||
if not policy_applies and not self._stream_should_retry(
|
||||
trigger, healthy_deployments, all_deployments, initial_kwargs
|
||||
):
|
||||
|
|
@ -9339,7 +9344,7 @@ class Router(DeprecatedRouterMembers):
|
|||
response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None)
|
||||
return response
|
||||
except Exception as e:
|
||||
if is_guardrail_intervention(e):
|
||||
if is_guardrail_intervention(e) or is_agentic_followup_failure(e):
|
||||
raise
|
||||
current_attempt = None
|
||||
original_exception = e
|
||||
|
|
@ -9446,6 +9451,8 @@ class Router(DeprecatedRouterMembers):
|
|||
return response
|
||||
|
||||
except Exception as e:
|
||||
if is_agentic_followup_failure(e):
|
||||
raise
|
||||
# Always track the latest error so we raise the most
|
||||
# recent exception instead of the first one.
|
||||
original_exception = e
|
||||
|
|
|
|||
|
|
@ -83,6 +83,7 @@ from litellm.constants import (
|
|||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
bind_budget_reservation_to_callbacks,
|
||||
normalize_drop_params,
|
||||
|
|
@ -2235,7 +2236,7 @@ def client(original_function):
|
|||
|
||||
is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
|
||||
if (
|
||||
num_retries and not is_completion_litellm_router_call
|
||||
num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e)
|
||||
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
|
||||
if (
|
||||
isinstance(e, openai.APIError)
|
||||
|
|
@ -2269,7 +2270,7 @@ def client(original_function):
|
|||
|
||||
is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
|
||||
if (
|
||||
num_retries and not is_responses_litellm_router_call
|
||||
num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e)
|
||||
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
|
||||
if (
|
||||
isinstance(e, openai.APIError)
|
||||
|
|
@ -2554,7 +2555,10 @@ def client(original_function):
|
|||
is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True)
|
||||
|
||||
if (
|
||||
num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError)
|
||||
num_retries
|
||||
and not is_acompletion_litellm_router_call
|
||||
and not isinstance(e, ImportError)
|
||||
and not is_agentic_followup_failure(e)
|
||||
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
|
||||
try:
|
||||
litellm.num_retries = None # set retries to None to prevent infinite loops
|
||||
|
|
@ -2585,7 +2589,10 @@ def client(original_function):
|
|||
is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True)
|
||||
|
||||
if (
|
||||
num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError)
|
||||
num_retries
|
||||
and not is_aresponses_litellm_router_call
|
||||
and not isinstance(e, ImportError)
|
||||
and not is_agentic_followup_failure(e)
|
||||
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
|
||||
try:
|
||||
litellm.num_retries = None # set retries to None to prevent infinite loops
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ they stay out of the body even without it.
|
|||
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Dict, Final, List, Optional, Tuple
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -264,8 +265,6 @@ class _GateOnlyLogger(CustomLogger):
|
|||
|
||||
|
||||
class _RecordingGateLogger(_GateOnlyLogger):
|
||||
"""Also keeps every follow-up response the loop hands to the post hook."""
|
||||
|
||||
def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None:
|
||||
super().__init__(plan=plan, tool_calls=tool_calls)
|
||||
self.followups: tuple[ModelResponse, ...] = ()
|
||||
|
|
@ -622,6 +621,35 @@ async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clien
|
|||
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refused_rerun_drops_a_call_claimed_only_by_name_next_to_one_claimed_by_id(restore_callbacks):
|
||||
response: Final = ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[
|
||||
_tool_call("call_abc", "litellm_code_execution"),
|
||||
_tool_call("call_search", "litellm_web_search"),
|
||||
_tool_call("call_client", "lookup"),
|
||||
],
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result: Final = await _run_refused_rerun(
|
||||
response,
|
||||
_REFUSING_RAILS[0],
|
||||
gate_tool_calls={"tool_calls": [{"id": "call_abc"}]},
|
||||
other_hooks_tool_calls=({"tool_calls": [{"name": "litellm_web_search"}]},),
|
||||
)
|
||||
|
||||
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"response, gate_tool_calls",
|
||||
|
|
@ -716,6 +744,118 @@ async def test_acompletion_bills_the_first_call_on_the_failure_log_when_the_foll
|
|||
)
|
||||
|
||||
|
||||
def _tool_call_turn(call_id: str, usage: Usage) -> ModelResponse:
|
||||
return ModelResponse(
|
||||
model="gpt-4o-mini",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message=Message(role="assistant", content=None, tool_calls=[_tool_call(call_id, "litellm_web_search")]),
|
||||
)
|
||||
],
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
|
||||
class _TwoRoundLogger(CustomLogger):
|
||||
def __init__(self, middle_round: ModelResponse) -> None:
|
||||
super().__init__()
|
||||
self._follow_ups: Final = (middle_round, "litellm.RateLimitError")
|
||||
self.logging_objs: tuple[object, ...] = ()
|
||||
|
||||
async def async_should_run_agentic_loop(
|
||||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> tuple[bool, dict[str, object]]:
|
||||
assert isinstance(response, ModelResponse)
|
||||
calls: Final = response.choices[0].message.tool_calls or []
|
||||
return bool(calls), {"tool_calls": [{"id": call.id} for call in calls]}
|
||||
|
||||
async def async_build_agentic_loop_plan(
|
||||
self,
|
||||
tools: Mapping[str, object],
|
||||
model: str,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
response: object,
|
||||
anthropic_messages_provider_config: object,
|
||||
anthropic_messages_optional_request_params: Mapping[str, object],
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> AgenticLoopPlan:
|
||||
follow_up: Final = self._follow_ups[len(self.logging_objs)]
|
||||
self.logging_objs = (*self.logging_objs, logging_obj)
|
||||
return AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": follow_up}),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_bills_every_served_round_on_the_failure_log_when_a_later_follow_up_fails(
|
||||
restore_callbacks, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
first_usage: Final = Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)
|
||||
middle_usage: Final = Usage(prompt_tokens=11, completion_tokens=4, total_tokens=15)
|
||||
logger: Final = _TwoRoundLogger(middle_round=_tool_call_turn("call_next", middle_usage))
|
||||
monkeypatch.setattr(litellm, "callbacks", [logger])
|
||||
mock_client: Final = MagicMock()
|
||||
mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_turn("call_abc", first_usage)))
|
||||
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await litellm.acompletion(
|
||||
model="openai/gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "what is 6*7?"}],
|
||||
api_key="sk-test",
|
||||
client=mock_client,
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
first_logging_obj, _ = logger.logging_objs
|
||||
assert isinstance(first_logging_obj, Logging)
|
||||
price: Final = litellm.model_cost["gpt-4o-mini"]
|
||||
billed_usage: Final = first_logging_obj.model_call_details["combined_usage_object"]
|
||||
assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (5 + 11, 3 + 4)
|
||||
assert first_logging_obj.model_call_details["response_cost"] == pytest.approx(
|
||||
(5 + 11) * price["input_cost_per_token"] + (3 + 4) * price["output_cost_per_token"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])
|
||||
async def test_a_follow_up_failing_with_a_builtin_error_reaches_the_caller_unchanged(stream: bool):
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
|
||||
|
||||
sandbox_error: Final = RuntimeError("sandbox crashed")
|
||||
|
||||
async def failing_follow_up() -> ModelResponse:
|
||||
raise sandbox_error
|
||||
|
||||
with pytest.raises(RuntimeError) as raised:
|
||||
await bill_first_round_on_failure(
|
||||
failing_follow_up(),
|
||||
logging_obj=Logging(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=stream,
|
||||
call_type="acompletion",
|
||||
start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
|
||||
litellm_call_id="call-builtin-error",
|
||||
function_id="fn-builtin-error",
|
||||
),
|
||||
response=_tool_call_turn("call_abc", Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
assert raised.value is sandbox_error
|
||||
|
||||
|
||||
class _RunOnlyLoopLogger(CustomLogger):
|
||||
def __init__(self, result: ModelResponse) -> None:
|
||||
super().__init__()
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import json
|
|||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
|
@ -5369,12 +5369,12 @@ class _PlannedChatLoop(CustomLogger):
|
|||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
) -> tuple[bool, dict]:
|
||||
kwargs: Mapping[str, object],
|
||||
) -> tuple[bool, dict[str, object]]:
|
||||
claimed: Final = [
|
||||
{"id": call.id, "name": call.function.name}
|
||||
for call in _tool_calls_of(response)
|
||||
|
|
@ -5384,14 +5384,14 @@ class _PlannedChatLoop(CustomLogger):
|
|||
|
||||
async def async_build_chat_completion_agentic_loop_plan(
|
||||
self,
|
||||
tools: dict,
|
||||
tools: Mapping[str, object],
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
response: object,
|
||||
optional_params: dict,
|
||||
optional_params: Mapping[str, object],
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: dict,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> AgenticLoopPlan:
|
||||
self.logging_objs.append(logging_obj)
|
||||
return self._plan
|
||||
|
|
@ -5406,24 +5406,24 @@ class _RunOnlyChatLoop(CustomLogger):
|
|||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
) -> tuple[bool, dict]:
|
||||
kwargs: Mapping[str, object],
|
||||
) -> tuple[bool, dict[str, object]]:
|
||||
return True, {"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]}
|
||||
|
||||
async def async_run_chat_completion_agentic_loop(
|
||||
self,
|
||||
tools: dict,
|
||||
tools: Mapping[str, object],
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
response: object,
|
||||
optional_params: dict,
|
||||
optional_params: Mapping[str, object],
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: dict,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> object:
|
||||
return self._result
|
||||
|
||||
|
|
@ -5599,12 +5599,12 @@ class _SearchAgainLoop(CustomLogger):
|
|||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
) -> tuple[bool, dict]:
|
||||
kwargs: Mapping[str, object],
|
||||
) -> tuple[bool, dict[str, object]]:
|
||||
searches: Final = [
|
||||
{"id": call.id, "name": call.function.name}
|
||||
for call in response.choices[0].message.tool_calls or []
|
||||
|
|
@ -5614,14 +5614,14 @@ class _SearchAgainLoop(CustomLogger):
|
|||
|
||||
async def async_build_chat_completion_agentic_loop_plan(
|
||||
self,
|
||||
tools: dict,
|
||||
tools: Mapping[str, object],
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
response: object,
|
||||
optional_params: dict,
|
||||
optional_params: Mapping[str, object],
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: dict,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> AgenticLoopPlan:
|
||||
return AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import gzip
|
|||
import importlib
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from types import ModuleType
|
||||
from typing import Final, Literal, Protocol
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
|
@ -236,12 +236,12 @@ class _SearchingChatGate(CustomLogger):
|
|||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
) -> tuple[bool, dict]:
|
||||
kwargs: Mapping[str, object],
|
||||
) -> tuple[bool, dict[str, object]]:
|
||||
return True, {"tool_calls": [{"id": "call-1"}]}
|
||||
|
||||
|
||||
|
|
@ -253,14 +253,14 @@ class _FailingFollowUpChatLoop(_SearchingChatGate):
|
|||
|
||||
async def async_run_chat_completion_agentic_loop(
|
||||
self,
|
||||
tools: dict,
|
||||
tools: Mapping[str, object],
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
response: object,
|
||||
optional_params: dict,
|
||||
optional_params: Mapping[str, object],
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: dict,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> object:
|
||||
self.runs += 1
|
||||
self.logging_objs.append(logging_obj)
|
||||
|
|
@ -272,26 +272,26 @@ class _RaisingAgenticGate(CustomLogger):
|
|||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
) -> tuple[bool, dict]:
|
||||
kwargs: Mapping[str, object],
|
||||
) -> tuple[bool, dict[str, object]]:
|
||||
raise ValueError("gate bug")
|
||||
|
||||
|
||||
class _AnsweringChatLoop(_SearchingChatGate):
|
||||
async def async_run_chat_completion_agentic_loop(
|
||||
self,
|
||||
tools: dict,
|
||||
tools: Mapping[str, object],
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
response: object,
|
||||
optional_params: dict,
|
||||
optional_params: Mapping[str, object],
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: dict,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> object:
|
||||
self.runs += 1
|
||||
return litellm.ModelResponse(
|
||||
|
|
@ -490,12 +490,12 @@ class _SearchAgainLoop(CustomLogger):
|
|||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
) -> tuple[bool, dict]:
|
||||
kwargs: Mapping[str, object],
|
||||
) -> tuple[bool, dict[str, object]]:
|
||||
searches: Final = [
|
||||
{"id": call.id, "name": call.function.name}
|
||||
for call in response.choices[0].message.tool_calls or []
|
||||
|
|
@ -505,14 +505,14 @@ class _SearchAgainLoop(CustomLogger):
|
|||
|
||||
async def async_build_chat_completion_agentic_loop_plan(
|
||||
self,
|
||||
tools: dict,
|
||||
tools: Mapping[str, object],
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
response: object,
|
||||
optional_params: dict,
|
||||
optional_params: Mapping[str, object],
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: dict,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> AgenticLoopPlan:
|
||||
return AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
|
|
|
|||
|
|
@ -4292,6 +4292,62 @@ class TestHandleLLMApiExceptionPricesWhatTheServerDispatched:
|
|||
assert resent >= dispatched_after_trim
|
||||
|
||||
|
||||
class TestHandleLLMApiExceptionReportsWhatAFailedAgenticFollowUpBills:
|
||||
async def _error_response_cost(self, raise_the_follow_up_error: bool) -> float:
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
processing_obj: Final = ProxyBaseLLMRequestProcessing(
|
||||
data={"model": "gpt-4o", "messages": [{"role": "user", "content": "what is 6*7?"}]}
|
||||
)
|
||||
caller: Final = ProxyUserAPIKeyAuth(request_route="/v1/chat/completions")
|
||||
mock_request: Final = MagicMock(spec=Request)
|
||||
mock_request.headers = {}
|
||||
proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache())
|
||||
proxy_logging.alert_types = []
|
||||
_, logging_obj = await processing_obj.common_processing_pre_call_logic(
|
||||
request=mock_request,
|
||||
general_settings={},
|
||||
user_api_key_dict=caller,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
proxy_config=MagicMock(spec=ProxyConfig),
|
||||
route_type="acompletion",
|
||||
)
|
||||
logging_obj.model_call_details.update(first_api_call_start_time=datetime.datetime.now())
|
||||
follow_up_error: Final = litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o")
|
||||
|
||||
async def failing_follow_up() -> ModelResponse:
|
||||
raise follow_up_error
|
||||
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await bill_first_round_on_failure(
|
||||
failing_follow_up(),
|
||||
logging_obj=logging_obj,
|
||||
response=ModelResponse(
|
||||
model="gpt-4o", usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672)
|
||||
),
|
||||
stream=False,
|
||||
)
|
||||
with pytest.raises(ProxyException) as raised:
|
||||
await processing_obj.handle_llm_api_exception(
|
||||
e=follow_up_error
|
||||
if raise_the_follow_up_error
|
||||
else litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o"),
|
||||
user_api_key_dict=caller,
|
||||
proxy_logging_obj=proxy_logging,
|
||||
)
|
||||
return float(raised.value.headers["x-litellm-response-cost"])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_error_response_reports_the_served_first_call_its_failure_row_bills(self) -> None:
|
||||
price: Final = litellm.model_cost["gpt-4o"]
|
||||
|
||||
assert await self._error_response_cost(raise_the_follow_up_error=True) == pytest.approx(
|
||||
559 * price["input_cost_per_token"] + 113 * price["output_cost_per_token"]
|
||||
)
|
||||
assert await self._error_response_cost(raise_the_follow_up_error=False) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestHandleLLMApiExceptionDictDetail:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3561,13 +3561,15 @@ def test_an_earlier_timed_out_responses_attempt_counts_its_input_from_the_newest
|
|||
[None, 1700000001.0],
|
||||
ids=["retry-not-dispatched", "retry-dispatched"],
|
||||
)
|
||||
def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attempts(
|
||||
def test_a_retry_failure_bills_input_only_for_dispatched_attempts(
|
||||
current_attempt_api_call_start_time: float | None,
|
||||
) -> None:
|
||||
from litellm.proxy.utils import _failure_fields_to_lift
|
||||
from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
prior_usage: Final = Usage(prompt_tokens=4, completion_tokens=2, total_tokens=6)
|
||||
estimated_retry_usage: Final = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None)
|
||||
assert estimated_retry_usage is not None
|
||||
|
||||
lifted: Final = _failure_fields_to_lift(
|
||||
{
|
||||
|
|
@ -3586,43 +3588,63 @@ def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attemp
|
|||
|
||||
usage: Final = lifted["combined_usage_object"]
|
||||
assert isinstance(usage, Usage)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (4, 2, 6)
|
||||
expected_retry_tokens: Final = (
|
||||
estimated_retry_usage.prompt_tokens if current_attempt_api_call_start_time is not None else 0
|
||||
)
|
||||
assert usage.prompt_tokens == prior_usage.prompt_tokens + expected_retry_tokens
|
||||
assert usage.completion_tokens == prior_usage.completion_tokens
|
||||
assert usage.total_tokens == prior_usage.total_tokens + expected_retry_tokens
|
||||
assert lifted["response_cost"] == 0.25
|
||||
|
||||
|
||||
def test_a_responses_failure_after_a_retried_follow_up_bills_only_the_first_calls_usage() -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_agentic_follow_up_failure_row_bills_each_served_first_call_without_an_estimate() -> None:
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure
|
||||
from litellm.proxy.utils import _failure_fields_to_lift
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
responses_input: Final = [{"role": "user", "content": "Search the web for the latest LiteLLM release"}]
|
||||
logging_obj: Final = Logging(
|
||||
model=FAILURE_USAGE_MODEL,
|
||||
messages=responses_input,
|
||||
messages=ONE_USER_MESSAGE,
|
||||
stream=False,
|
||||
call_type="aresponses",
|
||||
call_type="acompletion",
|
||||
start_time=real_datetime.datetime.now(),
|
||||
litellm_call_id="call-responses-retry",
|
||||
function_id="fn-responses-retry",
|
||||
litellm_call_id="call-agentic-follow-up",
|
||||
function_id="fn-agentic-follow-up",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model=FAILURE_USAGE_MODEL, user="", optional_params={}, litellm_params={"custom_llm_provider": "openai"}
|
||||
)
|
||||
logging_obj.pre_call(input=responses_input, api_key="sk-test")
|
||||
first_call: Final = ModelResponse(usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672))
|
||||
logging_obj.record_earlier_round_for_failure(first_call)
|
||||
first_call_cost: Final = logging_obj.model_call_details["response_cost"]
|
||||
logging_obj.carry_failed_attempt_usage()
|
||||
logging_obj.pre_call(input=responses_input, api_key="sk-test")
|
||||
|
||||
lifted: Final = _failure_fields_to_lift(
|
||||
{"litellm_logging_obj": logging_obj}, provider_billed=False, dispatched_call_type="aresponses"
|
||||
async def failing_follow_up() -> ModelResponse:
|
||||
raise litellm.RateLimitError(message="rate limited", llm_provider="openai", model=FAILURE_USAGE_MODEL)
|
||||
|
||||
async def serve_first_call_then_fail_its_follow_up(usage: Usage) -> float:
|
||||
logging_obj.pre_call(input=ONE_USER_MESSAGE, api_key="sk-test")
|
||||
with pytest.raises(litellm.RateLimitError):
|
||||
await bill_first_round_on_failure(
|
||||
failing_follow_up(),
|
||||
logging_obj=logging_obj,
|
||||
response=ModelResponse(model=FAILURE_USAGE_MODEL, usage=usage),
|
||||
stream=False,
|
||||
)
|
||||
return logging_obj.model_call_details["response_cost"]
|
||||
|
||||
primary_cost: Final = await serve_first_call_then_fail_its_follow_up(
|
||||
Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672)
|
||||
)
|
||||
logging_obj.carry_failed_attempt_usage()
|
||||
backup_cost: Final = await serve_first_call_then_fail_its_follow_up(
|
||||
Usage(prompt_tokens=31, completion_tokens=5, total_tokens=36)
|
||||
)
|
||||
|
||||
lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logging_obj})
|
||||
|
||||
usage: Final = lifted["combined_usage_object"]
|
||||
assert (usage.prompt_tokens, usage.completion_tokens) == (559, 113)
|
||||
assert first_call_cost > 0
|
||||
assert lifted["response_cost"] == pytest.approx(first_call_cost)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens) == (559 + 31, 113 + 5)
|
||||
assert primary_cost > 0 and backup_cost > 0
|
||||
assert lifted["response_cost"] == pytest.approx(primary_cost + backup_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging
|
|||
from litellm.llms.xai.common_utils import xai_reported_cost_in_usd
|
||||
from litellm.proxy.mcp_server import mcp_server_manager
|
||||
from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk
|
||||
from litellm.proxy.utils import _failure_fields_to_lift
|
||||
from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift
|
||||
from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler
|
||||
from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -540,13 +540,9 @@ async def test_mcp_allowed_tools_filtering():
|
|||
]
|
||||
|
||||
# First filter by allowed tools
|
||||
filtered_tools_with_duplicates = (
|
||||
LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
|
||||
mcp_tools=mock_mcp_tools_with_duplicates,
|
||||
mcp_tools_with_litellm_proxy=cast(
|
||||
List[ToolParam], mcp_tool_config_with_duplicates
|
||||
),
|
||||
)
|
||||
filtered_tools_with_duplicates = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
|
||||
mcp_tools=mock_mcp_tools_with_duplicates,
|
||||
mcp_tools_with_litellm_proxy=cast(List[ToolParam], mcp_tool_config_with_duplicates),
|
||||
)
|
||||
|
||||
# Then deduplicate the filtered tools
|
||||
|
|
@ -605,11 +601,9 @@ async def test_mcp_allowed_tools_filtering():
|
|||
},
|
||||
]
|
||||
|
||||
filtered_tools_multiple = (
|
||||
LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
|
||||
mcp_tools=mock_mcp_tools_from_server,
|
||||
mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs),
|
||||
)
|
||||
filtered_tools_multiple = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
|
||||
mcp_tools=mock_mcp_tools_from_server,
|
||||
mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs),
|
||||
)
|
||||
|
||||
# Should return union of all allowed tools (3 unique tools)
|
||||
|
|
@ -1918,7 +1912,7 @@ def _router_retrying_once() -> litellm.Router:
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds(
|
||||
async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
_serve_responses(
|
||||
|
|
@ -1945,9 +1939,11 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_ea
|
|||
assert len(recorder.logged) == 1
|
||||
row: Final = recorder.logged[0]
|
||||
assert row["id"] == response.id
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5)
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens)
|
||||
assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY))
|
||||
assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5)
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19)
|
||||
assert row["response_cost"] == pytest.approx(
|
||||
_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1982,7 +1978,22 @@ async def test_non_stream_auto_execute_failing_every_retry_bills_each_paid_first
|
|||
failure_row_fields: Final = _failure_fields_to_lift(MappingProxyType({"litellm_logging_obj": logging_obj}))
|
||||
billed_usage: Final = failure_row_fields["combined_usage_object"]
|
||||
assert isinstance(billed_usage, Usage)
|
||||
assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == billed_tokens
|
||||
estimated_failure_usage: Final = (
|
||||
_estimate_dispatched_failure_usage(
|
||||
model="gpt-5.6",
|
||||
request_input=logging_obj.model_call_details.get("messages"),
|
||||
system_input=None,
|
||||
)
|
||||
if not retried_first_rounds
|
||||
else None
|
||||
)
|
||||
expected_prompt_tokens: Final = billed_tokens[0] + (
|
||||
estimated_failure_usage.prompt_tokens if estimated_failure_usage is not None else 0
|
||||
)
|
||||
assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (
|
||||
expected_prompt_tokens,
|
||||
billed_tokens[1],
|
||||
)
|
||||
assert failure_row_fields["response_cost"] == pytest.approx(
|
||||
_round_cost(_ROUND_ONE_BODY) + sum(_round_cost(body) for body in retried_first_rounds)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3652,14 +3652,28 @@ class _CutSSE(httpx.AsyncByteStream):
|
|||
raise httpx.RemoteProtocolError("peer closed connection without sending complete message body")
|
||||
|
||||
|
||||
_BACKUP_SSE: Final = "".join(
|
||||
f"data: {json.dumps(chunk)}\n\n"
|
||||
for chunk in (
|
||||
{"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest",
|
||||
"choices": [{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}]},
|
||||
{"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest",
|
||||
"choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}},
|
||||
_BACKUP_SSE: Final = (
|
||||
"".join(
|
||||
f"data: {json.dumps(chunk)}\n\n"
|
||||
for chunk in (
|
||||
{
|
||||
"id": "b",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000,
|
||||
"model": "mistral-large-latest",
|
||||
"choices": [
|
||||
{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "b",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1700000000,
|
||||
"model": "mistral-large-latest",
|
||||
"choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96},
|
||||
},
|
||||
)
|
||||
)
|
||||
) + "data: [DONE]\n\n"
|
||||
|
||||
|
|
@ -3907,88 +3921,165 @@ class _ScriptedAgenticFollowUps(CustomLogger):
|
|||
self.shared_call_succeeded.set()
|
||||
|
||||
|
||||
async def _route_agentic_request_with_retries(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...]
|
||||
_AGENTIC_BACKUP_USAGE: Final = {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23}
|
||||
_AGENTIC_BACKUP_BODY: Final = {
|
||||
"id": "chatcmpl-backup",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "42"}}],
|
||||
"usage": _AGENTIC_BACKUP_USAGE,
|
||||
}
|
||||
_AGENTIC_RATE_LIMITED: Final = {
|
||||
"error": {"message": "Rate limit reached", "type": "requests", "code": "rate_limit_exceeded"}
|
||||
}
|
||||
|
||||
|
||||
async def _route_agentic_request(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
follow_ups: tuple[str, ...],
|
||||
model: str = "m",
|
||||
first_call_replies: tuple[httpx.Response, ...] = (),
|
||||
fallbacks: tuple[str, ...] = (),
|
||||
) -> tuple[object, LiteLLMLogging, respx.Route, _ScriptedAgenticFollowUps]:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
hook: Final = _ScriptedAgenticFollowUps(follow_ups)
|
||||
monkeypatch.setattr(litellm, "callbacks", [hook])
|
||||
first_calls: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY)
|
||||
first_calls: Final = (
|
||||
respx_mock.post("https://api.openai.com/v1/chat/completions").mock(side_effect=list(first_call_replies))
|
||||
if first_call_replies
|
||||
else respx_mock.post("https://api.openai.com/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY)
|
||||
)
|
||||
)
|
||||
respx_mock.post("https://backup.test/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_AGENTIC_BACKUP_BODY)
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}],
|
||||
model_list=[
|
||||
{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}},
|
||||
{
|
||||
"model_name": "backup",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "sk-test",
|
||||
"api_base": "https://backup.test/v1",
|
||||
},
|
||||
},
|
||||
],
|
||||
num_retries=2,
|
||||
retry_after=0,
|
||||
fallbacks=[{"m": list(fallbacks)}] if fallbacks else [],
|
||||
)
|
||||
messages: Final = [{"role": "user", "content": "what is 6*7?"}]
|
||||
shared_logging, _ = litellm.utils.function_setup(
|
||||
original_function="acompletion",
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=datetime.now(),
|
||||
model="m",
|
||||
model=model,
|
||||
messages=messages,
|
||||
litellm_call_id="shared-agentic-call",
|
||||
)
|
||||
try:
|
||||
outcome: object = await router.acompletion(
|
||||
model="m", messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging
|
||||
model=model, messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging
|
||||
)
|
||||
except litellm.RateLimitError as error:
|
||||
outcome = error
|
||||
return outcome, shared_logging, first_calls, hook
|
||||
|
||||
|
||||
def _gpt_4o_mini_cost(usage: Mapping[str, int]) -> float:
|
||||
return sum(
|
||||
litellm.cost_per_token(
|
||||
model="gpt-4o-mini", prompt_tokens=usage["prompt_tokens"], completion_tokens=usage["completion_tokens"]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_first_call(
|
||||
async def test_agentic_request_whose_follow_up_fails_is_not_retried_and_bills_its_one_first_call(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
from litellm.proxy.utils import _failure_fields_to_lift
|
||||
|
||||
outcome, shared_logging, first_calls, _ = await _route_agentic_request_with_retries(
|
||||
outcome, shared_logging, first_calls, _ = await _route_agentic_request(
|
||||
monkeypatch, respx_mock, ("litellm.RateLimitError",)
|
||||
)
|
||||
|
||||
lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": shared_logging})
|
||||
first_call_cost: Final = sum(
|
||||
litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7)
|
||||
)
|
||||
assert isinstance(outcome, litellm.RateLimitError)
|
||||
assert first_calls.call_count == 3
|
||||
assert lifted["combined_usage_object"].prompt_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"]
|
||||
assert lifted["combined_usage_object"].completion_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["completion_tokens"]
|
||||
assert lifted["response_cost"] == pytest.approx(3 * first_call_cost)
|
||||
|
||||
|
||||
async def _successful_agentic_request_cost_report(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...]
|
||||
) -> tuple[int, Mapping[str, object], Mapping[str, str | None]]:
|
||||
from litellm.proxy.common_request_processing import cost_headers
|
||||
|
||||
outcome, shared_logging, _, hook = await _route_agentic_request_with_retries(monkeypatch, respx_mock, follow_ups)
|
||||
await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10)
|
||||
assert isinstance(outcome, litellm.ModelResponse)
|
||||
success: Final = shared_logging.model_call_details["standard_logging_object"]
|
||||
billed: Final = {
|
||||
field: success[field] for field in ("response_cost", "prompt_tokens", "completion_tokens", "cost_breakdown")
|
||||
}
|
||||
return hook.plans_built, billed, cost_headers(success["response_cost"], shared_logging)
|
||||
assert outcome.status_code == 429
|
||||
assert first_calls.call_count == 1
|
||||
assert lifted["combined_usage_object"].prompt_tokens == _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"]
|
||||
assert lifted["combined_usage_object"].completion_tokens == _AGENTIC_FIRST_CALL_USAGE["completion_tokens"]
|
||||
assert lifted["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retried_agentic_request_that_recovers_reports_the_cost_of_a_first_try_success(
|
||||
async def test_agentic_request_whose_first_call_fails_is_still_retried(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
first_try_attempts, first_try_billed, first_try_headers = await _successful_agentic_request_cost_report(
|
||||
monkeypatch, respx_mock, ("42",)
|
||||
)
|
||||
recovered_attempts, recovered_billed, recovered_headers = await _successful_agentic_request_cost_report(
|
||||
monkeypatch, respx_mock, ("litellm.RateLimitError", "42")
|
||||
outcome, _, first_calls, hook = await _route_agentic_request(
|
||||
monkeypatch,
|
||||
respx_mock,
|
||||
("42",),
|
||||
first_call_replies=(
|
||||
httpx.Response(429, json=_AGENTIC_RATE_LIMITED),
|
||||
httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY),
|
||||
),
|
||||
)
|
||||
|
||||
assert (first_try_attempts, recovered_attempts) == (1, 2)
|
||||
assert recovered_billed == first_try_billed
|
||||
assert recovered_headers == first_try_headers
|
||||
assert isinstance(outcome, litellm.ModelResponse)
|
||||
assert first_calls.call_count == 2
|
||||
assert hook.plans_built == 1
|
||||
|
||||
|
||||
async def _successful_agentic_request_billing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
follow_ups: tuple[str, ...],
|
||||
model: str,
|
||||
fallbacks: tuple[str, ...] = (),
|
||||
) -> tuple[Mapping[str, object], Mapping[str, str | None]]:
|
||||
from litellm.proxy.common_request_processing import cost_headers
|
||||
|
||||
outcome, shared_logging, _, hook = await _route_agentic_request(
|
||||
monkeypatch, respx_mock, follow_ups, model=model, fallbacks=fallbacks
|
||||
)
|
||||
await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10)
|
||||
assert isinstance(outcome, litellm.ModelResponse)
|
||||
success: Final = shared_logging.model_call_details["standard_logging_object"]
|
||||
return success, cost_headers(success["response_cost"], shared_logging)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_after_a_failed_agentic_follow_up_bills_the_primarys_first_call_on_the_success_row(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
first_try, first_try_headers = await _successful_agentic_request_billing(
|
||||
monkeypatch, respx_mock, ("42",), model="backup"
|
||||
)
|
||||
recovered, recovered_headers = await _successful_agentic_request_billing(
|
||||
monkeypatch, respx_mock, ("litellm.RateLimitError",), model="m", fallbacks=("backup",)
|
||||
)
|
||||
|
||||
first_call_cost: Final = _gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE)
|
||||
assert first_try["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_BACKUP_USAGE))
|
||||
assert "failed_attempts_cost" not in first_try["cost_breakdown"].get("additional_costs", {})
|
||||
assert recovered["response_cost"] == pytest.approx(first_try["response_cost"] + first_call_cost)
|
||||
assert recovered["prompt_tokens"] == first_try["prompt_tokens"] + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"]
|
||||
assert (
|
||||
recovered["completion_tokens"]
|
||||
== first_try["completion_tokens"] + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"]
|
||||
)
|
||||
assert recovered["cost_breakdown"]["total_cost"] == pytest.approx(recovered["response_cost"])
|
||||
assert recovered["cost_breakdown"]["additional_costs"]["failed_attempts_cost"] == pytest.approx(first_call_cost)
|
||||
assert float(recovered_headers["x-litellm-response-cost"]) == pytest.approx(recovered["response_cost"])
|
||||
assert {key: value for key, value in recovered_headers.items() if key != "x-litellm-response-cost"} == {
|
||||
key: value for key, value in first_try_headers.items() if key != "x-litellm-response-cost"
|
||||
}
|
||||
|
||||
|
||||
def _responses_sse(*events: Mapping[str, object]) -> str:
|
||||
|
|
@ -13817,7 +13908,11 @@ def test_get_search_tool_support_reports_each_routable_deployment_of_the_group()
|
|||
@pytest.mark.parametrize("provider", ["chatgpt", "github_copilot"])
|
||||
@pytest.mark.parametrize("explicit_provider", [False, True])
|
||||
async def test_search_tool_catalog_lookup_needs_no_credentials(
|
||||
provider: str, explicit_provider: bool, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
provider: str,
|
||||
explicit_provider: bool,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "chatgpt"))
|
||||
monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path / "copilot"))
|
||||
|
|
@ -17485,13 +17580,16 @@ def test_model_group_info_fast_mode_fails_closed_without_explicit_boolean(
|
|||
def test_model_group_info_fast_mode_requires_every_deployment(
|
||||
local_model_cost_map: None, other_model: str, expected: bool, reverse: bool
|
||||
) -> None:
|
||||
models: Final = (other_model, "anthropic/claude-opus-5") if reverse else (
|
||||
"anthropic/claude-opus-5", other_model
|
||||
models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ("anthropic/claude-opus-5", other_model)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "fast-group",
|
||||
"litellm_params": {"model": model, "api_key": "fake-key"},
|
||||
}
|
||||
for model in models
|
||||
]
|
||||
)
|
||||
router: Final = Router(model_list=[{
|
||||
"model_name": "fast-group",
|
||||
"litellm_params": {"model": model, "api_key": "fake-key"},
|
||||
} for model in models])
|
||||
|
||||
result: Final = router.get_model_group_info("fast-group")
|
||||
|
||||
|
|
@ -17849,10 +17947,7 @@ class _AnthropicMessagesFallbackByteStream:
|
|||
|
||||
|
||||
def _anthropic_messages_overloaded_error_chunk() -> bytes:
|
||||
return (
|
||||
b"event: error\n"
|
||||
b'data: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n'
|
||||
)
|
||||
return b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n'
|
||||
|
||||
|
||||
def _anthropic_messages_invalid_request_error_chunk() -> bytes:
|
||||
|
|
@ -24600,16 +24695,25 @@ class TestAutoRouterTraceProvenance:
|
|||
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [logger, CallbackComplete()])
|
||||
classifier: Final = ObservedClassifier()
|
||||
router: Final = Router(model_list=[
|
||||
{**self._marker("blue"), "litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "custom", "classifier_plugin": classifier,
|
||||
"tiers": {"SIMPLE": "answer"}, "session_affinity": False, "deployment_affinity": False,
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
**self._marker("blue"),
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"complexity_router_config": {
|
||||
"classifier_type": "custom",
|
||||
"classifier_plugin": classifier,
|
||||
"tiers": {"SIMPLE": "answer"},
|
||||
"session_affinity": False,
|
||||
"deployment_affinity": False,
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
|
||||
], plugins=[classifier])
|
||||
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
|
||||
],
|
||||
plugins=[classifier],
|
||||
)
|
||||
stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"}
|
||||
request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale, "user_api_key_team_id": "team"}}
|
||||
with logger.tracer.start_as_current_span("request") as root:
|
||||
|
|
@ -24694,20 +24798,22 @@ class TestAutoRouterTraceProvenance:
|
|||
self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]"
|
||||
) -> None:
|
||||
logger, exporter = _tracing
|
||||
router: Final = Router(model_list=[
|
||||
self._marker("blue"),
|
||||
{
|
||||
"model_name": "traced-router",
|
||||
"model_info": {"id": "definition-semantic"},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/semantic-router",
|
||||
"tags": ["blue"],
|
||||
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
|
||||
"auto_router_default_model": "answer",
|
||||
"auto_router_embedding_model": "embed",
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
self._marker("blue"),
|
||||
{
|
||||
"model_name": "traced-router",
|
||||
"model_info": {"id": "definition-semantic"},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/semantic-router",
|
||||
"tags": ["blue"],
|
||||
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
|
||||
"auto_router_default_model": "answer",
|
||||
"auto_router_embedding_model": "embed",
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
]
|
||||
)
|
||||
with logger.start_phase_span("route traced-router"):
|
||||
response: Final = await router.async_pre_routing_hook(
|
||||
model="traced-router", request_kwargs={"metadata": {"tags": ["blue"]}}
|
||||
|
|
@ -24727,26 +24833,42 @@ class TestAutoRouterTraceProvenance:
|
|||
) -> None:
|
||||
logger, exporter = _tracing
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://embedding.example/v1/embeddings").respond(200, json={
|
||||
"object": "list",
|
||||
"model": "text-embedding-3-small",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
})
|
||||
router: Final = Router(model_list=[
|
||||
{
|
||||
"model_name": "internal-semantic-router",
|
||||
"model_info": {"id": "semantic-definition", "team_id": "trace-team", "team_public_model_name": "public-router"},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/semantic-router",
|
||||
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
|
||||
"auto_router_default_model": "answer",
|
||||
"auto_router_embedding_model": "embed",
|
||||
},
|
||||
respx_mock.post("https://embedding.example/v1/embeddings").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"model": "text-embedding-3-small",
|
||||
"data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
|
||||
"usage": {"prompt_tokens": 1, "total_tokens": 1},
|
||||
},
|
||||
{"model_name": "embed", "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", "api_key": "test"}},
|
||||
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
|
||||
])
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-semantic-router",
|
||||
"model_info": {
|
||||
"id": "semantic-definition",
|
||||
"team_id": "trace-team",
|
||||
"team_public_model_name": "public-router",
|
||||
},
|
||||
"litellm_params": {
|
||||
"model": "auto_router/semantic-router",
|
||||
"auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
|
||||
"auto_router_default_model": "answer",
|
||||
"auto_router_embedding_model": "embed",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "embed",
|
||||
"litellm_params": {
|
||||
"model": "openai/text-embedding-3-small",
|
||||
"api_base": "https://embedding.example/v1",
|
||||
"api_key": "test",
|
||||
},
|
||||
},
|
||||
{"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
|
||||
]
|
||||
)
|
||||
request: Final = {"metadata": {"user_api_key_team_id": "trace-team"}}
|
||||
with logger.tracer.start_as_current_span("request"):
|
||||
await router.async_get_available_deployment(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue